diff --git a/README.chromium b/README.chromium index 6192e3ffa..3ac3e8a0d 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1979 +Version: 1980 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/include/libyuv/cpu_id.h b/include/libyuv/cpu_id.h index ed814fc7b..3f50be7ad 100644 --- a/include/libyuv/cpu_id.h +++ b/include/libyuv/cpu_id.h @@ -56,7 +56,7 @@ static const int kCpuHasAVX512VL = 0x80000; static const int kCpuHasAVX512VNNI = 0x100000; static const int kCpuHasAVX512VBMI = 0x200000; static const int kCpuHasAVX512VBMI2 = 0x400000; -static const int kCpuHasAVX512VBITALG = 0x800000; +static const int kCpuHasAPX = 0x800000; static const int kCpuHasAVX10 = 0x1000000; static const int kCpuHasAVX10_2 = 0x2000000; static const int kCpuHasAVXVNNI = 0x4000000; diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index 63c6d665f..45af3e8c8 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -155,8 +155,7 @@ extern "C" { #endif // __clang__ // clang >= 7.0.0 required for AVX512. -#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && \ - (defined(__x86_64__) || defined(__i386__)) +#if defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) && defined(__x86_64__) // clang in xcode follows a different versioning scheme. // TODO(fbarchard): fix xcode 9 ios b/789. #if (__clang_major__ >= 7) && !defined(__APPLE__) diff --git a/include/libyuv/version.h b/include/libyuv/version.h index e3aa6940a..a6a1e6c34 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1979 +#define LIBYUV_VERSION 1980 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/source/cpu_id.cc b/source/cpu_id.cc index f39ecbab4..7af354697 100644 --- a/source/cpu_id.cc +++ b/source/cpu_id.cc @@ -32,6 +32,39 @@ #include // For getauxval() #endif +#if !defined(__pnacl__) && !defined(__CLR_VER) && defined(__linux__) && \ + defined(__x86_64__) +#include +#include +#ifndef ARCH_GET_XCOMP_PERM +#define ARCH_GET_XCOMP_PERM 0x1022 +#endif +#ifndef XFEATURE_XTILEDATA +#define XFEATURE_XTILEDATA 18 +#endif +// Linux 5.16+ disables AMX tile data (XTILEDATA) per process with XFD until +// the process requests it via arch_prctl(ARCH_REQ_XCOMP_PERM). Executing a +// tile instruction without permission raises SIGILL even though XCR0 reports +// tile state as enabled. libyuv only queries the permission and does not +// request it, so it never changes process state (signal frame size) or trips +// sandbox policies. Applications that want AMX/ACE kernels should request +// XTILEDATA permission before libyuv detects CPU features. +static int HasXTilePermission(void) { + unsigned long bitmask = 0; + if (syscall(SYS_arch_prctl, ARCH_GET_XCOMP_PERM, &bitmask) != 0) { + return 0; + } + return (bitmask & (1UL << XFEATURE_XTILEDATA)) != 0; +} +#elif !defined(__pnacl__) && !defined(__CLR_VER) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || \ + defined(_M_IX86)) +// Other operating systems enable tile state when XCR0 reports it. +static int HasXTilePermission(void) { + return 1; +} +#endif + #if defined(_WIN32) && defined(__aarch64__) #undef WIN32_LEAN_AND_MEAN #define WIN32_LEAN_AND_MEAN @@ -140,7 +173,7 @@ static int GetXCR0() { #if defined(_MSC_FULL_VER) && (_MSC_FULL_VER >= 160040219) xcr0 = (int)_xgetbv(0); // VS2010 SP1 required. NOLINT #elif defined(__i386__) || defined(__x86_64__) - asm(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0) : "c"(0) : "%edx"); + __asm__ __volatile__(".byte 0x0f, 0x01, 0xd0" : "=a"(xcr0) : "c"(0) : "edx"); #endif // defined(__i386__) || defined(__x86_64__) return xcr0; } @@ -516,6 +549,7 @@ static SAFEBUFFERS int GetCpuFlags(void) { ((cpu_info1[2] & 0x00080000) ? kCpuHasSSE41 : 0) | ((cpu_info1[2] & 0x00100000) ? kCpuHasSSE42 : 0) | ((cpu_info7[1] & 0x00000200) ? kCpuHasERMS : 0) | + ((cpu_amdinfo21[0] & 0x00008000) ? kCpuHasERMS : 0) | ((cpu_info7[3] & 0x00000010) ? kCpuHasFSMR : 0); // AVX requires OS saves YMM registers. @@ -525,25 +559,29 @@ static SAFEBUFFERS int GetCpuFlags(void) { ((cpu_info1[2] & 0x00001000) ? kCpuHasFMA3 : 0) | ((cpu_info1[2] & 0x20000000) ? kCpuHasF16C : 0) | ((cpu_einfo7[0] & 0x00000010) ? kCpuHasAVXVNNI : 0) | - ((cpu_einfo7[3] & 0x00000010) ? kCpuHasAVXVNNIINT8 : 0); + ((cpu_einfo7[3] & 0x00000010) ? kCpuHasAVXVNNIINT8 : 0) | + ((cpu_einfo7[3] & 0x00200000) ? kCpuHasAPX : 0); - cpu_info |= ((cpu_amdinfo21[0] & 0x00008000) ? kCpuHasERMS : 0); - - // Detect AVX512bw - if ((GetXCR0() & 0xe0) == 0xe0 && (cpu_info7[1] & 0x00010000)) { - cpu_info |= ((cpu_info7[1] & 0x40000000) ? kCpuHasAVX512BW : 0) | - ((cpu_info7[1] & 0x80000000) ? kCpuHasAVX512VL : 0) | - ((cpu_info7[2] & 0x00000002) ? kCpuHasAVX512VBMI : 0) | - ((cpu_info7[2] & 0x00000040) ? kCpuHasAVX512VBMI2 : 0) | - ((cpu_info7[2] & 0x00000800) ? kCpuHasAVX512VNNI : 0) | - ((cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0) | - ((cpu_einfo7[3] & 0x00080000) ? kCpuHasAVX10 : 0) | - ((cpu_info7[3] & 0x02000000) ? kCpuHasAMXINT8 : 0) | - ((cpu_info21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0) | - ((cpu_einfo1e[0] & 0x00000040) ? kCpuHasAMXTF32 : 0); + // Detect AVX512bw / AVX10 (512-bit vector support) + int has_avx512 = (cpu_info7[1] & 0x00010000) || + ((cpu_einfo7[3] & 0x00080000) && cpu_info0[0] >= 0x24 && + (cpu_info24[1] & 0x00040000)); + if ((GetXCR0() & 0xe0) == 0xe0 && has_avx512) { + int has_tile = ((GetXCR0() & 0x60000) == 0x60000) && + ((cpu_info7[3] & 0x01000000) != 0) && HasXTilePermission(); + cpu_info |= + ((cpu_info7[1] & 0x40000000) ? kCpuHasAVX512BW : 0) | + ((cpu_info7[1] & 0x80000000) ? kCpuHasAVX512VL : 0) | + ((cpu_info7[2] & 0x00000002) ? kCpuHasAVX512VBMI : 0) | + ((cpu_info7[2] & 0x00000040) ? kCpuHasAVX512VBMI2 : 0) | + ((cpu_info7[2] & 0x00000800) ? kCpuHasAVX512VNNI : 0) | + ((cpu_einfo7[3] & 0x00080000) ? kCpuHasAVX10 : 0) | + ((has_tile && (cpu_info7[3] & 0x02000000)) ? kCpuHasAMXINT8 : 0) | + ((cpu_amdinfo21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0) | + ((cpu_einfo1e[0] & 0x00000040) ? kCpuHasAMXTF32 : 0); if (cpu_info0[0] >= 0x24 && (cpu_einfo7[3] & 0x00080000)) { cpu_info |= ((cpu_info24[1] & 0xFF) >= 2) ? kCpuHasAVX10_2 : 0; - if ((GetXCR0() & 0x001600e6) == 0x001600e6 && + if (has_tile && (GetXCR0() & 0x001000e6) == 0x001000e6 && (cpu_einfo7[2] & 0x00000800) && (cpu_einfo1d_2[0] & 0xff) >= 1 && ((cpu_info24[1] & 0xff) >= 2 || ((cpu_info24[1] & 0xff) >= 1 && diff --git a/source/planar_functions.cc b/source/planar_functions.cc index ce4619b85..a00198673 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -60,6 +60,11 @@ void CopyPlane(const uint8_t* src_y, CopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; } #endif +#if defined(HAS_COPYROW_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + CopyRow = CopyRow_ERMS; + } +#endif #if defined(HAS_COPYROW_AVX) if (TestCpuFlag(kCpuHasAVX)) { CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX; @@ -70,11 +75,6 @@ void CopyPlane(const uint8_t* src_y, CopyRow = CopyRow_AVX512BW; } #endif -#if defined(HAS_COPYROW_ERMS) - if (TestCpuFlag(kCpuHasERMS)) { - CopyRow = CopyRow_ERMS; - } -#endif #if defined(HAS_COPYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON; @@ -5054,6 +5054,11 @@ void SetPlane(uint8_t* dst_y, } } #endif +#if defined(HAS_SETROW_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + SetRow = SetRow_ERMS; + } +#endif #if defined(HAS_SETROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { SetRow = SetRow_Any_AVX2; @@ -5062,11 +5067,6 @@ void SetPlane(uint8_t* dst_y, } } #endif -#if defined(HAS_SETROW_ERMS) - if (TestCpuFlag(kCpuHasERMS)) { - SetRow = SetRow_ERMS; - } -#endif #if defined(HAS_SETROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SetRow = SetRow_Any_LSX; @@ -6028,7 +6028,7 @@ int ARGBBlur(const uint8_t* src_argb, int32_t* max_cumsum_bot_row; int32_t* cumsum_top_row; - if (!src_argb || !dst_argb || width <= 0 || height == 0 || + if (!src_argb || !dst_argb || !dst_cumsum || width <= 0 || height == 0 || height == INT_MIN) { return -1; } @@ -6060,8 +6060,10 @@ int ARGBBlur(const uint8_t* src_argb, #endif // Compute enough CumulativeSum for first row to be blurred. After this // one row of CumulativeSum is updated at a time. - ARGBComputeCumulativeSum(src_argb, src_stride_argb, dst_cumsum, - dst_stride32_cumsum, width, radius); + if (ARGBComputeCumulativeSum(src_argb, src_stride_argb, dst_cumsum, + dst_stride32_cumsum, width, radius) != 0) { + return -1; + } src_argb = src_argb + radius * src_stride_argb; cumsum_bot_row = &dst_cumsum[(radius - 1) * dst_stride32_cumsum]; diff --git a/source/rotate.cc b/source/rotate.cc index a9f48d74c..c20a65b35 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -308,6 +308,11 @@ void RotatePlane180(const uint8_t* src, CopyRow = IS_ALIGNED(width, 32) ? CopyRow_SSE2 : CopyRow_Any_SSE2; } #endif +#if defined(HAS_COPYROW_ERMS) + if (TestCpuFlag(kCpuHasERMS)) { + CopyRow = CopyRow_ERMS; + } +#endif #if defined(HAS_COPYROW_AVX) if (TestCpuFlag(kCpuHasAVX)) { CopyRow = IS_ALIGNED(width, 64) ? CopyRow_AVX : CopyRow_Any_AVX; @@ -318,11 +323,6 @@ void RotatePlane180(const uint8_t* src, CopyRow = CopyRow_AVX512BW; } #endif -#if defined(HAS_COPYROW_ERMS) - if (TestCpuFlag(kCpuHasERMS)) { - CopyRow = CopyRow_ERMS; - } -#endif #if defined(HAS_COPYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { CopyRow = IS_ALIGNED(width, 32) ? CopyRow_NEON : CopyRow_Any_NEON; diff --git a/source/row_common.cc b/source/row_common.cc index 3bd708c5c..16d2a1810 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -3581,17 +3581,19 @@ void ComputeCumulativeSumRow_C(const uint8_t* row, int32_t* cumsum, const int32_t* previous_cumsum, int width) { - int32_t row_sum[4] = {0, 0, 0, 0}; + uint32_t row_sum[4] = {0, 0, 0, 0}; + uint32_t* u_cumsum = (uint32_t*)cumsum; + const uint32_t* u_previous_cumsum = (const uint32_t*)previous_cumsum; int x; for (x = 0; x < width; ++x) { row_sum[0] += row[x * 4 + 0]; row_sum[1] += row[x * 4 + 1]; row_sum[2] += row[x * 4 + 2]; row_sum[3] += row[x * 4 + 3]; - cumsum[x * 4 + 0] = row_sum[0] + previous_cumsum[x * 4 + 0]; - cumsum[x * 4 + 1] = row_sum[1] + previous_cumsum[x * 4 + 1]; - cumsum[x * 4 + 2] = row_sum[2] + previous_cumsum[x * 4 + 2]; - cumsum[x * 4 + 3] = row_sum[3] + previous_cumsum[x * 4 + 3]; + u_cumsum[x * 4 + 0] = row_sum[0] + u_previous_cumsum[x * 4 + 0]; + u_cumsum[x * 4 + 1] = row_sum[1] + u_previous_cumsum[x * 4 + 1]; + u_cumsum[x * 4 + 2] = row_sum[2] + u_previous_cumsum[x * 4 + 2]; + u_cumsum[x * 4 + 3] = row_sum[3] + u_previous_cumsum[x * 4 + 3]; } } @@ -3601,27 +3603,25 @@ void CumulativeSumToAverageRow_C(const int32_t* tl, int area, uint8_t* dst, int count) { + const uint32_t* u_tl = (const uint32_t*)tl; + const uint32_t* u_bl = (const uint32_t*)bl; float ooa; int i; assert(area != 0); ooa = 1.0f / STATIC_CAST(float, area); for (i = 0; i < count; ++i) { - dst[0] = - (uint8_t)(STATIC_CAST(float, bl[w + 0] + tl[0] - bl[0] - tl[w + 0]) * - ooa); - dst[1] = - (uint8_t)(STATIC_CAST(float, bl[w + 1] + tl[1] - bl[1] - tl[w + 1]) * - ooa); - dst[2] = - (uint8_t)(STATIC_CAST(float, bl[w + 2] + tl[2] - bl[2] - tl[w + 2]) * - ooa); - dst[3] = - (uint8_t)(STATIC_CAST(float, bl[w + 3] + tl[3] - bl[3] - tl[w + 3]) * - ooa); + uint32_t sum0 = u_bl[w + 0] + u_tl[0] - u_bl[0] - u_tl[w + 0]; + uint32_t sum1 = u_bl[w + 1] + u_tl[1] - u_bl[1] - u_tl[w + 1]; + uint32_t sum2 = u_bl[w + 2] + u_tl[2] - u_bl[2] - u_tl[w + 2]; + uint32_t sum3 = u_bl[w + 3] + u_tl[3] - u_bl[3] - u_tl[w + 3]; + dst[0] = (uint8_t)(STATIC_CAST(float, sum0) * ooa); + dst[1] = (uint8_t)(STATIC_CAST(float, sum1) * ooa); + dst[2] = (uint8_t)(STATIC_CAST(float, sum2) * ooa); + dst[3] = (uint8_t)(STATIC_CAST(float, sum3) * ooa); dst += 4; - tl += 4; - bl += 4; + u_tl += 4; + u_bl += 4; } } diff --git a/source/row_la.cc b/source/row_la.cc index 131781985..857086ac8 100644 --- a/source/row_la.cc +++ b/source/row_la.cc @@ -3691,9 +3691,9 @@ void DetileRow_16_LSX(const uint16_t* src, #endif // HAS_DETILEROW_16_LSX #ifdef HAS_MERGEARGBROW_LSX -void MergeARGBRow_LSX(const uint8_t* src_b, +void MergeARGBRow_LSX(const uint8_t* src_r, const uint8_t* src_g, - const uint8_t* src_r, + const uint8_t* src_b, const uint8_t* src_a, uint8_t* dst_argb, int width) { @@ -3731,9 +3731,9 @@ void MergeARGBRow_LSX(const uint8_t* src_b, #endif // HAS_MERGEARGBROW_LSX #ifdef HAS_MERGEXRGBROW_LSX -void MergeXRGBRow_LSX(const uint8_t* src_b, +void MergeXRGBRow_LSX(const uint8_t* src_r, const uint8_t* src_g, - const uint8_t* src_r, + const uint8_t* src_b, uint8_t* dst_argb, int width) { asm volatile( @@ -3770,9 +3770,9 @@ void MergeXRGBRow_LSX(const uint8_t* src_b, #ifdef HAS_SPLITARGBROW_LSX void SplitARGBRow_LSX(const uint8_t* src_argb, - uint8_t* dst_b, - uint8_t* dst_g, uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, uint8_t* dst_a, int width) { asm volatile( @@ -3810,9 +3810,9 @@ void SplitARGBRow_LSX(const uint8_t* src_argb, #ifdef HAS_SPLITXRGBROW_LSX void SplitXRGBRow_LSX(const uint8_t* src_argb, - uint8_t* dst_b, - uint8_t* dst_g, uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, int width) { asm volatile( "1:\n\t" @@ -8039,9 +8039,9 @@ void DetileRow_16_LASX(const uint16_t* src, #endif // HAS_DETILEROW_16_LASX #ifdef HAS_MERGEARGBROW_LASX -void MergeARGBRow_LASX(const uint8_t* src_b, +void MergeARGBRow_LASX(const uint8_t* src_r, const uint8_t* src_g, - const uint8_t* src_r, + const uint8_t* src_b, const uint8_t* src_a, uint8_t* dst_argb, int width) { @@ -8078,9 +8078,9 @@ void MergeARGBRow_LASX(const uint8_t* src_b, #endif // HAS_MERGEARGBROW_LASX #ifdef HAS_MERGEXRGBROW_LASX -void MergeXRGBRow_LASX(const uint8_t* src_b, +void MergeXRGBRow_LASX(const uint8_t* src_r, const uint8_t* src_g, - const uint8_t* src_r, + const uint8_t* src_b, uint8_t* dst_argb, int width) { asm volatile( @@ -8115,9 +8115,9 @@ void MergeXRGBRow_LASX(const uint8_t* src_b, #ifdef HAS_SPLITARGBROW_LASX void SplitARGBRow_LASX(const uint8_t* src_argb, - uint8_t* dst_b, - uint8_t* dst_g, uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, uint8_t* dst_a, int width) { asm volatile( @@ -8156,9 +8156,9 @@ void SplitARGBRow_LASX(const uint8_t* src_argb, #ifdef HAS_SPLITXRGBROW_LASX void SplitXRGBRow_LASX(const uint8_t* src_argb, - uint8_t* dst_b, - uint8_t* dst_g, uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, int width) { asm volatile( "1:\n\t" diff --git a/source/row_wasm.cc b/source/row_wasm.cc index 0ebc2326a..82717c41c 100644 --- a/source/row_wasm.cc +++ b/source/row_wasm.cc @@ -1469,9 +1469,9 @@ void SplitARGBRow_WASMSIMD(const uint8_t* src_argb, 11, 15, 19, 23, 27, 31); v128_t ra23 = wasm_i8x16_shuffle(s2, s3, 2, 6, 10, 14, 18, 22, 26, 30, 3, 7, 11, 15, 19, 23, 27, 31); - wasm_v128_store(dst_r, wasm_i64x2_shuffle(bg01, bg23, 0, 2)); + wasm_v128_store(dst_b, wasm_i64x2_shuffle(bg01, bg23, 0, 2)); wasm_v128_store(dst_g, wasm_i64x2_shuffle(bg01, bg23, 1, 3)); - wasm_v128_store(dst_b, wasm_i64x2_shuffle(ra01, ra23, 0, 2)); + wasm_v128_store(dst_r, wasm_i64x2_shuffle(ra01, ra23, 0, 2)); wasm_v128_store(dst_a, wasm_i64x2_shuffle(ra01, ra23, 1, 3)); src_argb += 64; dst_r += 16; @@ -1496,22 +1496,22 @@ void MergeARGBRow_WASMSIMD(const uint8_t* src_r, v128_t g = wasm_v128_load(src_g); v128_t b = wasm_v128_load(src_b); v128_t a = wasm_v128_load(src_a); - v128_t rg0 = wasm_i8x16_shuffle(r, g, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + v128_t bg0 = wasm_i8x16_shuffle(b, g, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); - v128_t rg1 = wasm_i8x16_shuffle(r, g, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + v128_t bg1 = wasm_i8x16_shuffle(b, g, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); - v128_t ba0 = wasm_i8x16_shuffle(b, a, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + v128_t ra0 = wasm_i8x16_shuffle(r, a, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); - v128_t ba1 = wasm_i8x16_shuffle(b, a, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + v128_t ra1 = wasm_i8x16_shuffle(r, a, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); wasm_v128_store(dst_argb, - wasm_i16x8_shuffle(rg0, ba0, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_i16x8_shuffle(bg0, ra0, 0, 8, 1, 9, 2, 10, 3, 11)); wasm_v128_store(dst_argb + 16, - wasm_i16x8_shuffle(rg0, ba0, 4, 12, 5, 13, 6, 14, 7, 15)); + wasm_i16x8_shuffle(bg0, ra0, 4, 12, 5, 13, 6, 14, 7, 15)); wasm_v128_store(dst_argb + 32, - wasm_i16x8_shuffle(rg1, ba1, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_i16x8_shuffle(bg1, ra1, 0, 8, 1, 9, 2, 10, 3, 11)); wasm_v128_store(dst_argb + 48, - wasm_i16x8_shuffle(rg1, ba1, 4, 12, 5, 13, 6, 14, 7, 15)); + wasm_i16x8_shuffle(bg1, ra1, 4, 12, 5, 13, 6, 14, 7, 15)); src_r += 16; src_g += 16; src_b += 16; @@ -1542,9 +1542,9 @@ void SplitXRGBRow_WASMSIMD(const uint8_t* src_argb, 10, 14, 18, 22, 26, 30); v128_t r23 = wasm_i8x16_shuffle(s2, s3, 2, 6, 10, 14, 18, 22, 26, 30, 2, 6, 10, 14, 18, 22, 26, 30); - wasm_v128_store(dst_r, wasm_i64x2_shuffle(bg01, bg23, 0, 2)); + wasm_v128_store(dst_b, wasm_i64x2_shuffle(bg01, bg23, 0, 2)); wasm_v128_store(dst_g, wasm_i64x2_shuffle(bg01, bg23, 1, 3)); - wasm_v128_store(dst_b, wasm_i64x2_shuffle(r01, r23, 0, 2)); + wasm_v128_store(dst_r, wasm_i64x2_shuffle(r01, r23, 0, 2)); src_argb += 64; dst_r += 16; dst_g += 16; @@ -1566,22 +1566,22 @@ void MergeXRGBRow_WASMSIMD(const uint8_t* src_r, v128_t r = wasm_v128_load(src_r); v128_t g = wasm_v128_load(src_g); v128_t b = wasm_v128_load(src_b); - v128_t rg0 = wasm_i8x16_shuffle(r, g, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + v128_t bg0 = wasm_i8x16_shuffle(b, g, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); - v128_t rg1 = wasm_i8x16_shuffle(r, g, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + v128_t bg1 = wasm_i8x16_shuffle(b, g, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); - v128_t ba0 = wasm_i8x16_shuffle(b, a, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, + v128_t ra0 = wasm_i8x16_shuffle(r, a, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23); - v128_t ba1 = wasm_i8x16_shuffle(b, a, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, + v128_t ra1 = wasm_i8x16_shuffle(r, a, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31); wasm_v128_store(dst_argb, - wasm_i16x8_shuffle(rg0, ba0, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_i16x8_shuffle(bg0, ra0, 0, 8, 1, 9, 2, 10, 3, 11)); wasm_v128_store(dst_argb + 16, - wasm_i16x8_shuffle(rg0, ba0, 4, 12, 5, 13, 6, 14, 7, 15)); + wasm_i16x8_shuffle(bg0, ra0, 4, 12, 5, 13, 6, 14, 7, 15)); wasm_v128_store(dst_argb + 32, - wasm_i16x8_shuffle(rg1, ba1, 0, 8, 1, 9, 2, 10, 3, 11)); + wasm_i16x8_shuffle(bg1, ra1, 0, 8, 1, 9, 2, 10, 3, 11)); wasm_v128_store(dst_argb + 48, - wasm_i16x8_shuffle(rg1, ba1, 4, 12, 5, 13, 6, 14, 7, 15)); + wasm_i16x8_shuffle(bg1, ra1, 4, 12, 5, 13, 6, 14, 7, 15)); src_r += 16; src_g += 16; src_b += 16; diff --git a/source/scale.cc b/source/scale.cc index 0a583e436..e6962fd22 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -1380,7 +1380,7 @@ static int ScalePlaneBilinearDown_16(int src_width, #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; - if (IS_ALIGNED(src_width, 32)) { + if (IS_ALIGNED(src_width, 16)) { InterpolateRow = InterpolateRow_16_AVX2; } } @@ -2056,7 +2056,7 @@ static int ScalePlaneBilinearUp_16(int src_width, #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; - if (IS_ALIGNED(dst_width, 32)) { + if (IS_ALIGNED(dst_width, 16)) { InterpolateRow = InterpolateRow_16_AVX2; } } @@ -2297,11 +2297,11 @@ int ScalePlane(const uint8_t* src, int dy = 0; int y = 0; // Check for 1 pixel and avoid FixedDiv overflow. - // TODO(b/573815252): Fix center row calculation when dst_height == 1 and - // src_height == 32768. if (dst_height == 1 && src_height >= 32768) { - dy = FixedDiv(src_height, src_height); - y = CENTERSTART(dy, -32768); + dy = 0; + y = (filtering == kFilterLinear || filtering == kFilterNone) + ? ((src_height / 2) << 16) + : (((src_height / 2) << 16) - 32768); } else if (dst_height <= src_height) { // When scaling down, use the center 2 rows to filter. dy = FixedDiv(src_height, dst_height); @@ -2418,11 +2418,11 @@ int ScalePlane_16(const uint16_t* src, int dy = 0; int y = 0; // Check for 1 pixel and avoid FixedDiv overflow. - // TODO(b/573815252): Fix center row calculation when dst_height == 1 and - // src_height == 32768. if (dst_height == 1 && src_height >= 32768) { - dy = FixedDiv(src_height, src_height); - y = CENTERSTART(dy, -32768); + dy = 0; + y = (filtering == kFilterLinear || filtering == kFilterNone) + ? ((src_height / 2) << 16) + : (((src_height / 2) << 16) - 32768); } else if (dst_height <= src_height) { // When scaling down, use the center 2 rows to filter. dy = FixedDiv(src_height, dst_height); diff --git a/source/scale_common.cc b/source/scale_common.cc index cdda834e8..94c709f36 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1731,7 +1731,7 @@ void ScalePlaneVertical_16(int src_height, #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; - if (IS_ALIGNED(dst_width_words, 32)) { + if (IS_ALIGNED(dst_width_words, 16)) { InterpolateRow = InterpolateRow_16_AVX2; } } @@ -1873,15 +1873,6 @@ void ScaleSlope(int src_width, assert(src_height != 0); assert(dst_width > 0); assert(dst_height > 0); - // Check for 1 pixel and avoid FixedDiv overflow. - // TODO(b/573815252): Fix center coordinate calculation when dst == 1 and - // src == 32768. - if (dst_width == 1 && Abs(src_width) >= 32768) { - dst_width = Abs(src_width); - } - if (dst_height == 1 && src_height >= 32768) { - dst_height = src_height; - } if (filtering == kFilterBox) { // Scale step for point sampling duplicates all pixels equally. *dx = FixedDiv(Abs(src_width), dst_width); @@ -1890,14 +1881,20 @@ void ScaleSlope(int src_width, *y = 0; } else if (filtering == kFilterBilinear) { // Scale step for bilinear sampling renders last pixel once for upsample. - if (dst_width <= Abs(src_width)) { + if (dst_width == 1 && Abs(src_width) >= 32768) { + *dx = 0; + *x = ((Abs(src_width) / 2) << 16) - 32768; + } else if (dst_width <= Abs(src_width)) { *dx = FixedDiv(Abs(src_width), dst_width); *x = CENTERSTART(*dx, -32768); // Subtract 0.5 (32768) to center filter. } else if (src_width > 1 && dst_width > 1) { *dx = FixedDiv1(Abs(src_width), dst_width); *x = 0; } - if (dst_height <= src_height) { + if (dst_height == 1 && src_height >= 32768) { + *dy = 0; + *y = ((src_height / 2) << 16) - 32768; + } else if (dst_height <= src_height) { *dy = FixedDiv(src_height, dst_height); *y = CENTERSTART(*dy, -32768); // Subtract 0.5 (32768) to center filter. } else if (src_height > 1 && dst_height > 1) { @@ -1906,21 +1903,39 @@ void ScaleSlope(int src_width, } } else if (filtering == kFilterLinear) { // Scale step for bilinear sampling renders last pixel once for upsample. - if (dst_width <= Abs(src_width)) { + if (dst_width == 1 && Abs(src_width) >= 32768) { + *dx = 0; + *x = (Abs(src_width) / 2) << 16; + } else if (dst_width <= Abs(src_width)) { *dx = FixedDiv(Abs(src_width), dst_width); *x = CENTERSTART(*dx, -32768); // Subtract 0.5 (32768) to center filter. } else if (src_width > 1 && dst_width > 1) { *dx = FixedDiv1(Abs(src_width), dst_width); *x = 0; } - *dy = FixedDiv(src_height, dst_height); - *y = *dy >> 1; + if (dst_height == 1 && src_height >= 32768) { + *dy = 0; + *y = (src_height / 2) << 16; + } else { + *dy = FixedDiv(src_height, dst_height); + *y = *dy >> 1; + } } else { // Scale step for point sampling duplicates all pixels equally. - *dx = FixedDiv(Abs(src_width), dst_width); - *dy = FixedDiv(src_height, dst_height); - *x = CENTERSTART(*dx, 0); - *y = CENTERSTART(*dy, 0); + if (dst_width == 1 && Abs(src_width) >= 32768) { + *dx = 0; + *x = (Abs(src_width) / 2) << 16; + } else { + *dx = FixedDiv(Abs(src_width), dst_width); + *x = CENTERSTART(*dx, 0); + } + if (dst_height == 1 && src_height >= 32768) { + *dy = 0; + *y = (src_height / 2) << 16; + } else { + *dy = FixedDiv(src_height, dst_height); + *y = CENTERSTART(*dy, 0); + } } // Negative src_width means horizontally mirror. if (src_width < 0) { diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index cde09b91b..b1ea10fbd 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -3352,6 +3352,9 @@ void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb, // Divide num by div and return as 16.16 fixed point result. int FixedDiv_X86(int num, int div) { +#if defined(__x86_64__) + return (int)(((int64_t)num * 65536) / div); +#else asm volatile( "cdq \n" "shld $0x10,%%eax,%%edx \n" @@ -3362,10 +3365,14 @@ int FixedDiv_X86(int num, int div) { : "c"(div) // %1 : "memory", "cc", "edx"); return num; +#endif } // Divide num - 1 by div - 1 and return as 16.16 fixed point result. int FixedDiv1_X86(int num, int div) { +#if defined(__x86_64__) + return (int)((((int64_t)num * 65536) - 0x00010001) / (div - 1)); +#else asm volatile( "cdq \n" "shld $0x10,%%eax,%%edx \n" @@ -3380,6 +3387,7 @@ int FixedDiv1_X86(int num, int div) { : : "memory", "cc", "edx"); return num; +#endif } #ifdef HAS_SCALEUVROWDOWN2_AVX2 diff --git a/unit_test/cpu_test.cc b/unit_test/cpu_test.cc index c6cd0ce74..4034b31f0 100644 --- a/unit_test/cpu_test.cc +++ b/unit_test/cpu_test.cc @@ -202,7 +202,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) { int has_avx512vnni = TestCpuFlag(kCpuHasAVX512VNNI); int has_avx512vbmi = TestCpuFlag(kCpuHasAVX512VBMI); int has_avx512vbmi2 = TestCpuFlag(kCpuHasAVX512VBMI2); - int has_avx512vbitalg = TestCpuFlag(kCpuHasAVX512VBITALG); + int has_apx = TestCpuFlag(kCpuHasAPX); int has_avx10 = TestCpuFlag(kCpuHasAVX10); int has_avx10_2 = TestCpuFlag(kCpuHasAVX10_2); int has_avxvnni = TestCpuFlag(kCpuHasAVXVNNI); @@ -227,7 +227,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) { printf("Has AVX512VNNI 0x%x\n", has_avx512vnni); printf("Has AVX512VBMI 0x%x\n", has_avx512vbmi); printf("Has AVX512VBMI2 0x%x\n", has_avx512vbmi2); - printf("Has AVX512VBITALG 0x%x\n", has_avx512vbitalg); + printf("Has APX 0x%x\n", has_apx); printf("Has AVX10 0x%x\n", has_avx10); printf("Has AVX10_2 0x%x\n", has_avx10_2); printf("HAS AVXVNNI 0x%x\n", has_avxvnni); diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 1d21e8c2d..18365a773 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -3482,25 +3482,20 @@ TEST_F(LibYUVPlanarTest, SwapUVPlane_Opt) { } TEST_F(LibYUVPlanarTest, MergeRGBPlane_Opt) { - // Round count up to multiple of 16 + // Merge the same C split planes with C and Opt so a channel order bug in + // MergeRGBRow cannot be masked by a matching bug in SplitRGBRow. const int kPixels = benchmark_width_ * benchmark_height_; align_buffer_page_end(src_pixels, kPixels * 3); align_buffer_page_end(tmp_pixels_c_r, kPixels); - align_buffer_page_end(tmp_pixels_opt_r, kPixels); align_buffer_page_end(tmp_pixels_c_g, kPixels); - align_buffer_page_end(tmp_pixels_opt_g, kPixels); align_buffer_page_end(tmp_pixels_c_b, kPixels); - align_buffer_page_end(tmp_pixels_opt_b, kPixels); align_buffer_page_end(dst_pixels_c, kPixels * 3); align_buffer_page_end(dst_pixels_opt, kPixels * 3); MemRandomize(src_pixels, kPixels * 3); MemRandomize(tmp_pixels_c_r, kPixels); - MemRandomize(tmp_pixels_opt_r, kPixels); MemRandomize(tmp_pixels_c_g, kPixels); - MemRandomize(tmp_pixels_opt_g, kPixels); MemRandomize(tmp_pixels_c_b, kPixels); - MemRandomize(tmp_pixels_opt_b, kPixels); MemRandomize(dst_pixels_c, kPixels * 3); MemRandomize(dst_pixels_opt, kPixels * 3); @@ -3515,14 +3510,9 @@ TEST_F(LibYUVPlanarTest, MergeRGBPlane_Opt) { benchmark_height_); MaskCpuFlags(benchmark_cpu_info_); - SplitRGBPlane(src_pixels, benchmark_width_ * 3, tmp_pixels_opt_r, - benchmark_width_, tmp_pixels_opt_g, benchmark_width_, - tmp_pixels_opt_b, benchmark_width_, benchmark_width_, - benchmark_height_); - for (int i = 0; i < benchmark_iterations_; ++i) { - MergeRGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g, - benchmark_width_, tmp_pixels_opt_b, benchmark_width_, + MergeRGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g, + benchmark_width_, tmp_pixels_c_b, benchmark_width_, dst_pixels_opt, benchmark_width_ * 3, benchmark_width_, benchmark_height_); } @@ -3533,11 +3523,8 @@ TEST_F(LibYUVPlanarTest, MergeRGBPlane_Opt) { free_aligned_buffer_page_end(src_pixels); free_aligned_buffer_page_end(tmp_pixels_c_r); - free_aligned_buffer_page_end(tmp_pixels_opt_r); free_aligned_buffer_page_end(tmp_pixels_c_g); - free_aligned_buffer_page_end(tmp_pixels_opt_g); free_aligned_buffer_page_end(tmp_pixels_c_b); - free_aligned_buffer_page_end(tmp_pixels_opt_b); free_aligned_buffer_page_end(dst_pixels_c); free_aligned_buffer_page_end(dst_pixels_opt); } @@ -3587,6 +3574,13 @@ TEST_F(LibYUVPlanarTest, SplitRGBPlane_Opt) { dst_pixels_opt, benchmark_width_ * 3, benchmark_width_, benchmark_height_); + // Compare the split planes directly. A round trip alone would hide a + // channel swap that is mirrored in MergeRGBRow. + for (int i = 0; i < kPixels; ++i) { + ASSERT_EQ(tmp_pixels_c_r[i], tmp_pixels_opt_r[i]); + ASSERT_EQ(tmp_pixels_c_g[i], tmp_pixels_opt_g[i]); + ASSERT_EQ(tmp_pixels_c_b[i], tmp_pixels_opt_b[i]); + } for (int i = 0; i < kPixels * 3; ++i) { ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); } @@ -3621,28 +3615,22 @@ TEST_F(LibYUVPlanarTest, SplitRGBPlaneOverflow) { } TEST_F(LibYUVPlanarTest, MergeARGBPlane_Opt) { + // Merge the same C split planes with C and Opt so a channel order bug in + // MergeARGBRow cannot be masked by a matching bug in SplitARGBRow. const int kPixels = benchmark_width_ * benchmark_height_; align_buffer_page_end(src_pixels, kPixels * 4); align_buffer_page_end(tmp_pixels_c_r, kPixels); - align_buffer_page_end(tmp_pixels_opt_r, kPixels); align_buffer_page_end(tmp_pixels_c_g, kPixels); - align_buffer_page_end(tmp_pixels_opt_g, kPixels); align_buffer_page_end(tmp_pixels_c_b, kPixels); - align_buffer_page_end(tmp_pixels_opt_b, kPixels); align_buffer_page_end(tmp_pixels_c_a, kPixels); - align_buffer_page_end(tmp_pixels_opt_a, kPixels); align_buffer_page_end(dst_pixels_c, kPixels * 4); align_buffer_page_end(dst_pixels_opt, kPixels * 4); MemRandomize(src_pixels, kPixels * 4); MemRandomize(tmp_pixels_c_r, kPixels); - MemRandomize(tmp_pixels_opt_r, kPixels); MemRandomize(tmp_pixels_c_g, kPixels); - MemRandomize(tmp_pixels_opt_g, kPixels); MemRandomize(tmp_pixels_c_b, kPixels); - MemRandomize(tmp_pixels_opt_b, kPixels); MemRandomize(tmp_pixels_c_a, kPixels); - MemRandomize(tmp_pixels_opt_a, kPixels); MemRandomize(dst_pixels_opt, kPixels * 4); MemRandomize(dst_pixels_c, kPixels * 4); @@ -3657,15 +3645,10 @@ TEST_F(LibYUVPlanarTest, MergeARGBPlane_Opt) { benchmark_width_ * 4, benchmark_width_, benchmark_height_); MaskCpuFlags(benchmark_cpu_info_); - SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r, - benchmark_width_, tmp_pixels_opt_g, benchmark_width_, - tmp_pixels_opt_b, benchmark_width_, tmp_pixels_opt_a, - benchmark_width_, benchmark_width_, benchmark_height_); - for (int i = 0; i < benchmark_iterations_; ++i) { - MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g, - benchmark_width_, tmp_pixels_opt_b, benchmark_width_, - tmp_pixels_opt_a, benchmark_width_, dst_pixels_opt, + MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g, + benchmark_width_, tmp_pixels_c_b, benchmark_width_, + tmp_pixels_c_a, benchmark_width_, dst_pixels_opt, benchmark_width_ * 4, benchmark_width_, benchmark_height_); } @@ -3675,13 +3658,9 @@ TEST_F(LibYUVPlanarTest, MergeARGBPlane_Opt) { free_aligned_buffer_page_end(src_pixels); free_aligned_buffer_page_end(tmp_pixels_c_r); - free_aligned_buffer_page_end(tmp_pixels_opt_r); free_aligned_buffer_page_end(tmp_pixels_c_g); - free_aligned_buffer_page_end(tmp_pixels_opt_g); free_aligned_buffer_page_end(tmp_pixels_c_b); - free_aligned_buffer_page_end(tmp_pixels_opt_b); free_aligned_buffer_page_end(tmp_pixels_c_a); - free_aligned_buffer_page_end(tmp_pixels_opt_a); free_aligned_buffer_page_end(dst_pixels_opt); free_aligned_buffer_page_end(dst_pixels_c); } @@ -3735,6 +3714,14 @@ TEST_F(LibYUVPlanarTest, SplitARGBPlane_Opt) { tmp_pixels_opt_a, benchmark_width_, dst_pixels_opt, benchmark_width_ * 4, benchmark_width_, benchmark_height_); + // Compare the split planes directly. A round trip alone would hide a + // channel swap that is mirrored in MergeARGBRow. + for (int i = 0; i < kPixels; ++i) { + ASSERT_EQ(tmp_pixels_c_r[i], tmp_pixels_opt_r[i]); + ASSERT_EQ(tmp_pixels_c_g[i], tmp_pixels_opt_g[i]); + ASSERT_EQ(tmp_pixels_c_b[i], tmp_pixels_opt_b[i]); + ASSERT_EQ(tmp_pixels_c_a[i], tmp_pixels_opt_a[i]); + } for (int i = 0; i < kPixels * 4; ++i) { ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); } @@ -3772,24 +3759,20 @@ TEST_F(LibYUVPlanarTest, SplitARGBPlaneOverflow) { } TEST_F(LibYUVPlanarTest, MergeXRGBPlane_Opt) { + // Merge the same C split planes with C and Opt so a channel order bug in + // MergeXRGBRow cannot be masked by a matching bug in SplitXRGBRow. const int kPixels = benchmark_width_ * benchmark_height_; align_buffer_page_end(src_pixels, kPixels * 4); align_buffer_page_end(tmp_pixels_c_r, kPixels); - align_buffer_page_end(tmp_pixels_opt_r, kPixels); align_buffer_page_end(tmp_pixels_c_g, kPixels); - align_buffer_page_end(tmp_pixels_opt_g, kPixels); align_buffer_page_end(tmp_pixels_c_b, kPixels); - align_buffer_page_end(tmp_pixels_opt_b, kPixels); align_buffer_page_end(dst_pixels_c, kPixels * 4); align_buffer_page_end(dst_pixels_opt, kPixels * 4); MemRandomize(src_pixels, kPixels * 4); MemRandomize(tmp_pixels_c_r, kPixels); - MemRandomize(tmp_pixels_opt_r, kPixels); MemRandomize(tmp_pixels_c_g, kPixels); - MemRandomize(tmp_pixels_opt_g, kPixels); MemRandomize(tmp_pixels_c_b, kPixels); - MemRandomize(tmp_pixels_opt_b, kPixels); MemRandomize(dst_pixels_c, kPixels * 4); MemRandomize(dst_pixels_opt, kPixels * 4); @@ -3804,15 +3787,10 @@ TEST_F(LibYUVPlanarTest, MergeXRGBPlane_Opt) { benchmark_height_); MaskCpuFlags(benchmark_cpu_info_); - SplitARGBPlane(src_pixels, benchmark_width_ * 4, tmp_pixels_opt_r, - benchmark_width_, tmp_pixels_opt_g, benchmark_width_, - tmp_pixels_opt_b, benchmark_width_, NULL, 0, benchmark_width_, - benchmark_height_); - for (int i = 0; i < benchmark_iterations_; ++i) { - MergeARGBPlane(tmp_pixels_opt_r, benchmark_width_, tmp_pixels_opt_g, - benchmark_width_, tmp_pixels_opt_b, benchmark_width_, NULL, - 0, dst_pixels_opt, benchmark_width_ * 4, benchmark_width_, + MergeARGBPlane(tmp_pixels_c_r, benchmark_width_, tmp_pixels_c_g, + benchmark_width_, tmp_pixels_c_b, benchmark_width_, NULL, 0, + dst_pixels_opt, benchmark_width_ * 4, benchmark_width_, benchmark_height_); } @@ -3822,11 +3800,8 @@ TEST_F(LibYUVPlanarTest, MergeXRGBPlane_Opt) { free_aligned_buffer_page_end(src_pixels); free_aligned_buffer_page_end(tmp_pixels_c_r); - free_aligned_buffer_page_end(tmp_pixels_opt_r); free_aligned_buffer_page_end(tmp_pixels_c_g); - free_aligned_buffer_page_end(tmp_pixels_opt_g); free_aligned_buffer_page_end(tmp_pixels_c_b); - free_aligned_buffer_page_end(tmp_pixels_opt_b); free_aligned_buffer_page_end(dst_pixels_c); free_aligned_buffer_page_end(dst_pixels_opt); } @@ -3876,6 +3851,13 @@ TEST_F(LibYUVPlanarTest, SplitXRGBPlane_Opt) { dst_pixels_opt, benchmark_width_ * 4, benchmark_width_, benchmark_height_); + // Compare the split planes directly. A round trip alone would hide a + // channel swap that is mirrored in MergeXRGBRow. + for (int i = 0; i < kPixels; ++i) { + ASSERT_EQ(tmp_pixels_c_r[i], tmp_pixels_opt_r[i]); + ASSERT_EQ(tmp_pixels_c_g[i], tmp_pixels_opt_g[i]); + ASSERT_EQ(tmp_pixels_c_b[i], tmp_pixels_opt_b[i]); + } for (int i = 0; i < kPixels * 4; ++i) { ASSERT_EQ(dst_pixels_c[i], dst_pixels_opt[i]); } @@ -3891,6 +3873,91 @@ TEST_F(LibYUVPlanarTest, SplitXRGBPlane_Opt) { free_aligned_buffer_page_end(dst_pixels_opt); } +// Check absolute channel order for packed <-> planar RGB conversions using +// known byte values, independent of the C reference and of round trips. +// libyuv ARGB/XRGB are B, G, R, A in memory; RGB (MergeRGBPlane) is R, G, B. +TEST_F(LibYUVPlanarTest, SplitMergePlane_ChannelOrder) { + const int kWidth = 64 * 3 + 7; // Exercise SIMD main loops and tails. + align_buffer_page_end(packed, kWidth * 4); + align_buffer_page_end(plane_r, kWidth); + align_buffer_page_end(plane_g, kWidth); + align_buffer_page_end(plane_b, kWidth); + align_buffer_page_end(plane_a, kWidth); + align_buffer_page_end(dst, kWidth * 4); + + MaskCpuFlags(benchmark_cpu_info_); + + // ARGB: memory order B, G, R, A. + for (int i = 0; i < kWidth; ++i) { + packed[i * 4 + 0] = (uint8_t)(i * 4 + 0); // B + packed[i * 4 + 1] = (uint8_t)(i * 4 + 1); // G + packed[i * 4 + 2] = (uint8_t)(i * 4 + 2); // R + packed[i * 4 + 3] = (uint8_t)(i * 4 + 3); // A + } + SplitARGBPlane(packed, kWidth * 4, plane_r, kWidth, plane_g, kWidth, plane_b, + kWidth, plane_a, kWidth, kWidth, 1); + for (int i = 0; i < kWidth; ++i) { + ASSERT_EQ((uint8_t)(i * 4 + 0), plane_b[i]) << "SplitARGB B " << i; + ASSERT_EQ((uint8_t)(i * 4 + 1), plane_g[i]) << "SplitARGB G " << i; + ASSERT_EQ((uint8_t)(i * 4 + 2), plane_r[i]) << "SplitARGB R " << i; + ASSERT_EQ((uint8_t)(i * 4 + 3), plane_a[i]) << "SplitARGB A " << i; + } + memset(dst, 0, kWidth * 4); + MergeARGBPlane(plane_r, kWidth, plane_g, kWidth, plane_b, kWidth, plane_a, + kWidth, dst, kWidth * 4, kWidth, 1); + for (int i = 0; i < kWidth * 4; ++i) { + ASSERT_EQ(packed[i], dst[i]) << "MergeARGB byte " << i; + } + + // XRGB: dst_a == NULL selects SplitXRGBRow / MergeXRGBRow. + memset(plane_r, 0, kWidth); + memset(plane_g, 0, kWidth); + memset(plane_b, 0, kWidth); + SplitARGBPlane(packed, kWidth * 4, plane_r, kWidth, plane_g, kWidth, plane_b, + kWidth, NULL, 0, kWidth, 1); + for (int i = 0; i < kWidth; ++i) { + ASSERT_EQ((uint8_t)(i * 4 + 0), plane_b[i]) << "SplitXRGB B " << i; + ASSERT_EQ((uint8_t)(i * 4 + 1), plane_g[i]) << "SplitXRGB G " << i; + ASSERT_EQ((uint8_t)(i * 4 + 2), plane_r[i]) << "SplitXRGB R " << i; + } + memset(dst, 0, kWidth * 4); + MergeARGBPlane(plane_r, kWidth, plane_g, kWidth, plane_b, kWidth, NULL, 0, + dst, kWidth * 4, kWidth, 1); + for (int i = 0; i < kWidth; ++i) { + ASSERT_EQ(packed[i * 4 + 0], dst[i * 4 + 0]) << "MergeXRGB B " << i; + ASSERT_EQ(packed[i * 4 + 1], dst[i * 4 + 1]) << "MergeXRGB G " << i; + ASSERT_EQ(packed[i * 4 + 2], dst[i * 4 + 2]) << "MergeXRGB R " << i; + ASSERT_EQ(255, dst[i * 4 + 3]) << "MergeXRGB A " << i; + } + + // RGB: memory order R, G, B. + for (int i = 0; i < kWidth; ++i) { + packed[i * 3 + 0] = (uint8_t)(i * 3 + 0); // R + packed[i * 3 + 1] = (uint8_t)(i * 3 + 1); // G + packed[i * 3 + 2] = (uint8_t)(i * 3 + 2); // B + } + SplitRGBPlane(packed, kWidth * 3, plane_r, kWidth, plane_g, kWidth, plane_b, + kWidth, kWidth, 1); + for (int i = 0; i < kWidth; ++i) { + ASSERT_EQ((uint8_t)(i * 3 + 0), plane_r[i]) << "SplitRGB R " << i; + ASSERT_EQ((uint8_t)(i * 3 + 1), plane_g[i]) << "SplitRGB G " << i; + ASSERT_EQ((uint8_t)(i * 3 + 2), plane_b[i]) << "SplitRGB B " << i; + } + memset(dst, 0, kWidth * 3); + MergeRGBPlane(plane_r, kWidth, plane_g, kWidth, plane_b, kWidth, dst, + kWidth * 3, kWidth, 1); + for (int i = 0; i < kWidth * 3; ++i) { + ASSERT_EQ(packed[i], dst[i]) << "MergeRGB byte " << i; + } + + free_aligned_buffer_page_end(packed); + free_aligned_buffer_page_end(plane_r); + free_aligned_buffer_page_end(plane_g); + free_aligned_buffer_page_end(plane_b); + free_aligned_buffer_page_end(plane_a); + free_aligned_buffer_page_end(dst); +} + // Merge 4 channels #define TESTQPLANARTOPI(FUNC, STYPE, DTYPE, DEPTH, W1280, N, NEG, OFF) \ TEST_F(LibYUVPlanarTest, FUNC##Plane_##DEPTH##N) { \ diff --git a/unit_test/unit_test.cc b/unit_test/unit_test.cc index d51eeec60..be00172f6 100644 --- a/unit_test/unit_test.cc +++ b/unit_test/unit_test.cc @@ -197,8 +197,8 @@ static int TestCpuEnv(int cpu_info) { if (TestEnv("LIBYUV_DISABLE_AVX512VBMI2")) { cpu_info &= ~libyuv::kCpuHasAVX512VBMI2; } - if (TestEnv("LIBYUV_DISABLE_AVX512VBITALG")) { - cpu_info &= ~libyuv::kCpuHasAVX512VBITALG; + if (TestEnv("LIBYUV_DISABLE_APX")) { + cpu_info &= ~libyuv::kCpuHasAPX; } if (TestEnv("LIBYUV_DISABLE_AVX10")) { cpu_info &= ~libyuv::kCpuHasAVX10; diff --git a/util/cpuid.c b/util/cpuid.c index 2ae73bb45..5980689e5 100644 --- a/util/cpuid.c +++ b/util/cpuid.c @@ -212,7 +212,7 @@ int main(int argc, const char* argv[]) { int has_avx512vnni = TestCpuFlag(kCpuHasAVX512VNNI); int has_avx512vbmi = TestCpuFlag(kCpuHasAVX512VBMI); int has_avx512vbmi2 = TestCpuFlag(kCpuHasAVX512VBMI2); - int has_avx512vbitalg = TestCpuFlag(kCpuHasAVX512VBITALG); + int has_apx = TestCpuFlag(kCpuHasAPX); int has_avx10 = TestCpuFlag(kCpuHasAVX10); int has_avx10_2 = TestCpuFlag(kCpuHasAVX10_2); int has_avxvnni = TestCpuFlag(kCpuHasAVXVNNI); @@ -237,7 +237,7 @@ int main(int argc, const char* argv[]) { printf("Has AVX512VNNI 0x%x\n", has_avx512vnni); printf("Has AVX512VBMI 0x%x\n", has_avx512vbmi); printf("Has AVX512VBMI2 0x%x\n", has_avx512vbmi2); - printf("Has AVX512VBITALG 0x%x\n", has_avx512vbitalg); + printf("Has APX 0x%x\n", has_apx); printf("Has AVX10 0x%x\n", has_avx10); printf("Has AVX10_2 0x%x\n", has_avx10_2); printf("HAS AVXVNNI 0x%x\n", has_avxvnni);