coffee/libyuv
1
0
Fork 0
mirror of https://chromium.googlesource.com/libyuv/libyuv synced 2026-10-29 02:17:13 +08:00
Code Issues Packages Projects Releases Wiki Activity
libyuv/source/scale_uv.cc
Frank Barchard f249eaa72f [libyuv] Fix top 20 bugs and performance issues 
1. Fix multi-bit TestCpuFlag(A | B) checks in AYUVToNV12, AYUVToNV21,
   RAWToRGBA, NV12ToRGB24Matrix, NV21ToRGB24Matrix, MergeRGBPlane, and
   MergeXR30Plane to compare == (A | B) so AVX-512 VBMI/BW kernels are
   not dispatched when AVX512VBMI or AVX512VL is absent.
2. Fix 2-byte SVE2 heap buffer overflow on odd widths in
   I422ToARGB1555Row_SVE_SC and I422ToARGB4444Row_SVE_SC using
   trn1/trn2 + st2b under .b predicates.
3. Fix ARGBPolynomialRow_SSE2 to use packssdw before packuswb, and add
   missing vzeroupper to SplitRGBRow_AVX2.
4. Fix CalcFrameSsim to return 0.0 instead of 0.0 / 0 (NaN) when
   width <= 8 || height <= 8 so I420Ssim stays finite on small frames.
5. Fix negative-height vertical inversion in ConvertFromI420,
   SplitARGBPlane (when dst_a is NULL), I422Rotate, I010Rotate, and
   I210Rotate.
6. Fix horizontal flip (negative x/dx) in ScaleFilterCols_NEON (signed
   sbfx #16, #16 offset extraction) and ScaleAddCols1_16_C (shift
   x >>= 16 before indexing src_ptr).
7. Fix ConvertToARGB odd crop_x chroma alignment in NV12/NV21, src_u
   offset in I420/YV12/J420/H420/U420, and FOURCC_U422 calling
   U422ToARGB instead of H422ToARGB.
8. Remove dead 14-instruction block in ScaleRowUp2_Bilinear_16_SSE2.
9. Fix UVScale_16 for src_width == 1 (P010ToP410/P210ToP410), enable
   and fix column order in TransposeWx4_32_SVE2, fix ByteToFloatRow
   NEON Any BPP (3 -> 4), fix AB64ToARGBRow_Any_*, ScaleColsUp2_SSE2,
   ScaleUVRowDown2Box_SME, and UYVYToARGBRow_SME macro guards, fix SVE2
   loop tail branches (b.lt/b.ge), and remove duplicate HAS_* macros.
10. Fix ConvertFP16ToFP32Column_NEON loop condition and unsigned b.lo
    underflow on widths < 8 or not multiples of 8.
11. Fix UYVYToNV12 IS_ALIGNED(awidth, ...) checks and WASM SIMD
    IS_ALIGNED(width, 16) checks in I420ToRGB24Matrix /
    I422ToRGB24Matrix.
12. Wire SplitUVRow_SVE2, SplitUVRow_LSX, and SplitUVRow_LASX in
    NV24ToARGB.
13. Fix MJpegDecoder heap buffer overflows in LoadFrame (reallocate
    databuf_[i] when scanlines_size changes), DecodeToBuffers, and
    DecodeToCallback (clamp scanlines_to_copy to lines_left and align
    skip to max_v_samp_factor).
14. Replace width & (vl - 1) tail calculations with post-loop remainder
    across 18 SVE2 functions in row_sve.h for non-power-of-2 SVE VL,
    and remove duplicate label 2: in UYVYToARGBRow_SVE_SC.
15. Handle width & 15 tail pixels in DetileToYUY2_C and validate
    IS_POWEROFTWO(tile_height) in DetileSplitUVPlane and DetileToYUY2.
16. Avoid signed 32-bit overflow in DivideRow_16_C and C16TO8, fix
    ClampFloatToZero_SSE2 pointer increment (lea 0x4) and loop counter
    (sub $0x1), and initialize xmm3 in ARGBCopyYToAlphaRow_SSE2.
17. Check max leaf 7 subleaf and max extended CPUID leaf in GetCpuFlags
    before querying subleaf 7:1 and extended leaf 0x80000021.
18. Free plane_u on error in I422ToNV21, and fix 4x/2x scratch row
    over-allocation in ScaleARGBBilinearDown and ScaleUVBilinearDown.
19. Add ARGBSetRow_RVV and ARGBShuffleRow_RVV in row_rvv.cc.
20. Fix NEON mask (15 -> 7) in ScaleUVRowUp2_Linear_Any_NEON /
    _16_Any_NEON, use SIMD InterpolateRow + SplitUVRow in NV16ToI420,
    wire SSSE3/SSE2 in ARGBToI400Matrix / ARGBToYUY2Matrix /
    ARGBToUYVYMatrix and RGB24Mirror, and split HalfFloatRow multiply
    into (src * scale) * 2^-112 to avoid 50x Intel denormal stalls.

Bug: b/574714081
Test: libyuv_unittest
Change-Id: Iec3ae5c58d52bee3ef13cf9b765ac84b733aa576
Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8601931
Reviewed-by: richard winterton <rrwinterton@gmail.com>
Commit-Queue: Frank Barchard <fbarchard@google.com>
2026-10-19 16:46:40 -07:00

1482 lines
47 KiB
C++
Raw Blame History

/*
* Copyright 2020 The LibYuv Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "libyuv/scale_uv.h"
#include <assert.h>
#include <limits.h>
#include <string.h>
#include "libyuv/cpu_id.h"
#include "libyuv/planar_functions.h" // For CopyUV
#include "libyuv/row.h"
#include "libyuv/scale_row.h"
#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif
// Macros to enable specialized scalers
#ifndef HAS_SCALEUVDOWN2
#define HAS_SCALEUVDOWN2 1
#endif
#ifndef HAS_SCALEUVDOWN4BOX
#define HAS_SCALEUVDOWN4BOX 1
#endif
#ifndef HAS_SCALEUVDOWNEVEN
#define HAS_SCALEUVDOWNEVEN 1
#endif
#ifndef HAS_SCALEUVBOX
#define HAS_SCALEUVBOX 1
#endif
#ifndef HAS_SCALEUVBILINEARDOWN
#define HAS_SCALEUVBILINEARDOWN 1
#endif
#ifndef HAS_SCALEUVBILINEARUP
#define HAS_SCALEUVBILINEARUP 1
#endif
#ifndef HAS_UVCOPY
#define HAS_UVCOPY 1
#endif
#ifndef HAS_SCALEPLANEVERTICAL
#define HAS_SCALEPLANEVERTICAL 1
#endif
#define MIN1(x) ((x) < 1 ? 1 : (x))
// ScaleUV, 1/2
// This is an optimized version for scaling down a UV to 1/2 of
// its original size.
#if HAS_SCALEUVDOWN2
static void ScaleUVDown2(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_uv,
uint8_t* dst_uv,
int x,
int dx,
int y,
int dy,
enum FilterMode filtering) {
int j;
ptrdiff_t row_stride = src_stride * (dy >> 16);
void (*ScaleUVRowDown2)(const uint8_t* src_uv, ptrdiff_t src_stride,
uint8_t* dst_uv, int dst_width) =
filtering == kFilterNone
? ScaleUVRowDown2_C
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_C
: ScaleUVRowDown2Box_C);
(void)src_width;
(void)src_height;
(void)dx;
assert(dx == 65536 * 2); // Test scale factor of 2.
assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2.
// Advance to odd row, even column.
if (filtering == kFilterBilinear) {
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
} else {
src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2;
}
#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && filtering) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_SSSE3;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_SSSE3;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWN2BOX_AVX2)
if (TestCpuFlag(kCpuHasAVX2) && filtering) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_AVX2;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWN2_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVRowDown2 =
filtering == kFilterNone
? ScaleUVRowDown2_Any_NEON
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_NEON
: ScaleUVRowDown2Box_Any_NEON);
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVRowDown2 =
filtering == kFilterNone
? ScaleUVRowDown2_NEON
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_NEON
: ScaleUVRowDown2Box_NEON);
}
}
#endif
#if defined(HAS_SCALEUVROWDOWN2_SME)
if (TestCpuFlag(kCpuHasSME)) {
ScaleUVRowDown2 = filtering == kFilterNone ? ScaleUVRowDown2_SME
: filtering == kFilterLinear ? ScaleUVRowDown2Linear_SME
: ScaleUVRowDown2Box_SME;
}
#endif
#if defined(HAS_SCALEUVROWDOWN2_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleUVRowDown2 =
filtering == kFilterNone
? ScaleUVRowDown2_RVV
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_RVV
: ScaleUVRowDown2Box_RVV);
}
#endif
#if defined(HAS_SCALEUVROWDOWN2_HVX)
if (TestCpuFlag(kCpuHasHVX)) {
ScaleUVRowDown2 =
filtering == kFilterNone
? ScaleUVRowDown2_HVX
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_HVX
: ScaleUVRowDown2Box_HVX);
}
#endif
// This code is not enabled. Only box filter is available at this time.
#if defined(HAS_SCALEUVROWDOWN2_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ScaleUVRowDown2 =
filtering == kFilterNone
? ScaleUVRowDown2_Any_SSSE3
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_SSSE3
: ScaleUVRowDown2Box_Any_SSSE3);
if (IS_ALIGNED(dst_width, 2)) {
ScaleUVRowDown2 =
filtering == kFilterNone
? ScaleUVRowDown2_SSSE3
: (filtering == kFilterLinear ? ScaleUVRowDown2Linear_SSSE3
: ScaleUVRowDown2Box_SSSE3);
}
}
#endif
if (filtering == kFilterLinear) {
src_stride = 0;
}
for (j = 0; j < dst_height; ++j) {
ScaleUVRowDown2(src_uv, src_stride, dst_uv, dst_width);
src_uv += row_stride;
dst_uv += dst_stride;
}
}
#endif // HAS_SCALEUVDOWN2
// ScaleUV, 1/4
// This is an optimized version for scaling down a UV to 1/4 of
// its original size.
#if HAS_SCALEUVDOWN4BOX
static int ScaleUVDown4Box(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_uv,
uint8_t* dst_uv,
int x,
int dx,
int y,
int dy) {
int j;
// Allocate 2 rows of UV.
const int row_size = (dst_width * 2 * 2 + 15) & ~15;
align_buffer_64(row, row_size * 2);
if (!row)
return 1;
ptrdiff_t row_stride = src_stride * (dy >> 16);
void (*ScaleUVRowDown2)(const uint8_t* src_uv, ptrdiff_t src_stride,
uint8_t* dst_uv, int dst_width) =
ScaleUVRowDown2Box_C;
// Advance to odd row, even column.
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
(void)src_width;
(void)src_height;
(void)dx;
assert(dx == 65536 * 4); // Test scale factor of 4.
assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4.
#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_SSSE3;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_SSSE3;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWN2BOX_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_AVX2;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWN2BOX_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_NEON;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWN2_SME)
if (TestCpuFlag(kCpuHasSME)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_SME;
}
#endif
#if defined(HAS_SCALEUVROWDOWN2BOX_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_RVV;
}
#endif
#if defined(HAS_SCALEUVROWDOWN2BOX_HVX)
if (TestCpuFlag(kCpuHasHVX)) {
ScaleUVRowDown2 = ScaleUVRowDown2Box_HVX;
}
#endif
for (j = 0; j < dst_height; ++j) {
ScaleUVRowDown2(src_uv, src_stride, row, dst_width * 2);
ScaleUVRowDown2(src_uv + src_stride * 2, src_stride, row + row_size,
dst_width * 2);
ScaleUVRowDown2(row, row_size, dst_uv, dst_width);
src_uv += row_stride;
dst_uv += dst_stride;
}
free_aligned_buffer_64(row);
return 0;
}
#endif // HAS_SCALEUVDOWN4BOX
// ScaleUV, any dimensions, box filter.
// The counterpart of ScalePlaneBox in scale.cc. Rows are summed into 16 bit
// accumulators, then each destination pixel averages its box of source pixels.
#if HAS_SCALEUVBOX
static void ScaleUVAddCols1_C(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_uv,
uint8_t* dst_uv) {
int i, j;
int boxwidth = MIN1(dx >> 16);
int scaleval = 65536 / (boxwidth * boxheight);
x >>= 16;
for (i = 0; i < dst_width; ++i) {
int u = 0;
int v = 0;
for (j = 0; j < boxwidth; ++j) {
u += src_uv[(x + j) * 2];
v += src_uv[(x + j) * 2 + 1];
}
x += boxwidth;
*dst_uv++ = (uint8_t)(u * scaleval >> 16);
*dst_uv++ = (uint8_t)(v * scaleval >> 16);
}
}
static void ScaleUVAddCols2_C(int dst_width,
int boxheight,
int x,
int dx,
const uint16_t* src_uv,
uint8_t* dst_uv) {
int i, j;
int minboxwidth = dx >> 16;
int scaletbl[2];
scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight);
scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight);
for (i = 0; i < dst_width; ++i) {
int ix = x >> 16;
int u = 0;
int v = 0;
int boxwidth;
int scaleval;
x += dx;
boxwidth = MIN1((x >> 16) - ix);
assert((boxwidth - minboxwidth == 0) || (boxwidth - minboxwidth == 1));
scaleval = scaletbl[boxwidth - minboxwidth];
for (j = 0; j < boxwidth; ++j) {
u += src_uv[(ix + j) * 2];
v += src_uv[(ix + j) * 2 + 1];
}
*dst_uv++ = (uint8_t)(u * scaleval >> 16);
*dst_uv++ = (uint8_t)(v * scaleval >> 16);
}
}
static int ScaleUVBox(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_uv,
uint8_t* dst_uv,
int x,
int dx,
int y,
int dy) {
int j, k;
const uint32_t max_y = (uint32_t)src_height << 16;
// Sized on src_width to keep ScaleAddRow aligned.
const int row_samples = src_width * 2;
{
// Row accumulator, two uint16_t per source pixel.
align_buffer_64(row16, row_samples * 2);
if (!row16)
return 1;
{
void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx,
const uint16_t* src_uv, uint8_t* dst_uv) =
(dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C;
void (*ScaleUVAddCols3)(int dst_width, int scaleval,
const uint16_t* src_uv, uint8_t* dst_uv) = NULL;
// Summing a row is position wise, so an interleaved UV row can be fed to
// the plane accumulator as src_width * 2 samples.
void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr,
int src_width) = ScaleAddRow_C;
if (dx == 0x30000) {
ScaleUVAddCols3 = ScaleUVAddCols3_C;
#if defined(HAS_SCALEUVADDCOLS3_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_AVX2;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_AVX2;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_AVX512VBMI)
if (TestCpuFlag(kCpuHasAVX512VBMI)) {
ScaleUVAddCols3 = ScaleUVAddCols3_AVX512VBMI;
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_NEON;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_SME)
if (TestCpuFlag(kCpuHasSME)) {
ScaleUVAddCols3 = ScaleUVAddCols3_SME;
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleUVAddCols3 = ScaleUVAddCols3_RVV;
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_LSX;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_LSX;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_WASMSIMD)
if (TestCpuFlag(kCpuHasWASM)) {
ScaleUVAddCols3 = ScaleUVAddCols3_Any_WASMSIMD;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVAddCols3 = ScaleUVAddCols3_WASMSIMD;
}
}
#endif
#if defined(HAS_SCALEUVADDCOLS3_HVX)
if (TestCpuFlag(kCpuHasHVX)) {
ScaleUVAddCols3 = ScaleUVAddCols3_HVX;
}
#endif
}
#if defined(HAS_SCALEADDROW_SSE2)
if (TestCpuFlag(kCpuHasSSE2)) {
ScaleAddRow = ScaleAddRow_Any_SSE2;
if (IS_ALIGNED(row_samples, 16)) {
ScaleAddRow = ScaleAddRow_SSE2;
}
}
#endif
#if defined(HAS_SCALEADDROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
ScaleAddRow = ScaleAddRow_Any_AVX2;
if (IS_ALIGNED(row_samples, 32)) {
ScaleAddRow = ScaleAddRow_AVX2;
}
}
#endif
#if defined(HAS_SCALEADDROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
ScaleAddRow = ScaleAddRow_AVX512BW;
}
#endif
#if defined(HAS_SCALEADDROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleAddRow = ScaleAddRow_Any_NEON;
if (IS_ALIGNED(row_samples, 16)) {
ScaleAddRow = ScaleAddRow_NEON;
}
}
#endif
#if defined(HAS_SCALEADDROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
ScaleAddRow = ScaleAddRow_Any_LSX;
if (IS_ALIGNED(row_samples, 16)) {
ScaleAddRow = ScaleAddRow_LSX;
}
}
#endif
#if defined(HAS_SCALEADDROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
ScaleAddRow = ScaleAddRow_RVV;
}
#endif
#if defined(HAS_SCALEADDROW_HVX)
if (TestCpuFlag(kCpuHasHVX)) {
ScaleAddRow = ScaleAddRow_HVX;
}
#endif
#if defined(HAS_SCALEADDROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASM)) {
ScaleAddRow = ScaleAddRow_Any_WASMSIMD;
if (IS_ALIGNED(row_samples, 16)) {
ScaleAddRow = ScaleAddRow_WASMSIMD;
}
}
#endif
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int boxheight;
int iy = uy >> 16;
const uint8_t* src = src_uv + iy * src_stride;
uy += dy;
if (uy > max_y) {
uy = max_y;
}
boxheight = MIN1((int)(uy >> 16) - iy);
memset(row16, 0, row_samples * 2);
for (k = 0; k < boxheight; ++k) {
ScaleAddRow(src, (uint16_t*)(row16), row_samples);
src += src_stride;
}
if (ScaleUVAddCols3 && boxheight <= 85) {
ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight),
(const uint16_t*)row16 + (ptrdiff_t)(x >> 16) * 2,
dst_uv);
} else {
ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16),
dst_uv);
}
dst_uv += dst_stride;
}
}
free_aligned_buffer_64(row16);
}
return 0;
}
#endif // HAS_SCALEUVBOX
// ScaleUV Even
// This is an optimized version for scaling down a UV to even
// multiple of its original size.
#if HAS_SCALEUVDOWNEVEN
static void ScaleUVDownEven(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_uv,
uint8_t* dst_uv,
int x,
int dx,
int y,
int dy,
enum FilterMode filtering) {
int j;
int col_step = dx >> 16;
ptrdiff_t row_stride = (dy >> 16) * src_stride;
void (*ScaleUVRowDownEven)(const uint8_t* src_uv, ptrdiff_t src_stride,
int src_step, uint8_t* dst_uv, int dst_width) =
filtering ? ScaleUVRowDownEvenBox_C : ScaleUVRowDownEven_C;
(void)src_width;
(void)src_height;
assert(IS_ALIGNED(src_width, 2));
assert(IS_ALIGNED(src_height, 2));
src_uv += (y >> 16) * src_stride + (x >> 16) * 2;
#if defined(HAS_SCALEUVROWDOWNEVEN_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3)) {
ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_SSSE3
: ScaleUVRowDownEven_Any_SSSE3;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVRowDownEven =
filtering ? ScaleUVRowDownEvenBox_SSE2 : ScaleUVRowDownEven_SSSE3;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWNEVEN_NEON)
if (TestCpuFlag(kCpuHasNEON) && !filtering) {
ScaleUVRowDownEven = ScaleUVRowDownEven_Any_NEON;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVRowDownEven = ScaleUVRowDownEven_NEON;
}
}
#endif // TODO(fbarchard): Enable Box filter
#if defined(HAS_SCALEUVROWDOWNEVENBOX_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_NEON
: ScaleUVRowDownEven_Any_NEON;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVRowDownEven =
filtering ? ScaleUVRowDownEvenBox_NEON : ScaleUVRowDownEven_NEON;
}
}
#endif
#if defined(HAS_SCALEUVROWDOWNEVEN_RVV) || defined(HAS_SCALEUVROWDOWN4_RVV)
if (TestCpuFlag(kCpuHasRVV) && !filtering) {
#if defined(HAS_SCALEUVROWDOWNEVEN_RVV)
ScaleUVRowDownEven = ScaleUVRowDownEven_RVV;
#endif
#if defined(HAS_SCALEUVROWDOWN4_RVV)
if (col_step == 4) {
ScaleUVRowDownEven = ScaleUVRowDown4_RVV;
}
#endif
}
#endif
if (filtering == kFilterLinear) {
src_stride = 0;
}
for (j = 0; j < dst_height; ++j) {
ScaleUVRowDownEven(src_uv, src_stride, col_step, dst_uv, dst_width);
src_uv += row_stride;
dst_uv += dst_stride;
}
}
#endif
// Scale UV down with bilinear interpolation.
#if HAS_SCALEUVBILINEARDOWN
static int ScaleUVBilinearDown(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_uv,
uint8_t* dst_uv,
int x,
int dx,
int y,
int dy,
enum FilterMode filtering) {
int j;
void (*InterpolateRow)(uint8_t* dst_uv, const uint8_t* src_uv,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
void (*ScaleUVFilterCols)(uint8_t* dst_uv, const uint8_t* src_uv,
int dst_width, int x, int dx) =
(src_width >= 32768) ? ScaleUVFilterCols64_C : ScaleUVFilterCols_C;
int64_t xlast = x + (int64_t)(dst_width - 1) * dx;
int64_t xl = (dx >= 0) ? x : xlast;
int64_t xr = (dx >= 0) ? xlast : x;
int clip_src_width;
xl = (xl >> 16) & ~3; // Left edge aligned.
xr = (xr >> 16) + 1; // Right most pixel used. Bilinear uses 2 pixels.
xr = (xr + 1 + 3) & ~3; // 1 beyond 4 pixel aligned right most pixel.
if (xr > src_width) {
xr = src_width;
}
clip_src_width = (int)(xr - xl) * 2; // Width aligned to 2.
src_uv += xl * 2;
x -= (int)(xl << 16);
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
if (IS_ALIGNED(clip_src_width, 32)) {
InterpolateRow = InterpolateRow_AVX2;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
if (IS_ALIGNED(clip_src_width, 16)) {
InterpolateRow = InterpolateRow_NEON;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
InterpolateRow = InterpolateRow_SVE2;
}
#endif
#if defined(HAS_INTERPOLATEROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
InterpolateRow = InterpolateRow_SME;
}
#endif
#if defined(HAS_INTERPOLATEROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
InterpolateRow = InterpolateRow_Any_LSX;
if (IS_ALIGNED(clip_src_width, 16)) {
InterpolateRow = InterpolateRow_LSX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(clip_src_width, 32)) {
InterpolateRow = InterpolateRow_LASX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
InterpolateRow = InterpolateRow_RVV;
}
#endif
#if defined(HAS_INTERPOLATEROW_HVX)
if (TestCpuFlag(kCpuHasHVX)) {
InterpolateRow = InterpolateRow_HVX;
}
#endif
#if defined(HAS_INTERPOLATEROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
InterpolateRow = InterpolateRow_Any_WASMSIMD;
if (IS_ALIGNED(clip_src_width, 16)) {
InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
#if defined(HAS_SCALEUVFILTERCOLS_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleUVFilterCols = ScaleUVFilterCols_SSSE3;
}
#endif
#if defined(HAS_SCALEUVFILTERCOLS_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON;
if (IS_ALIGNED(dst_width, 4)) {
ScaleUVFilterCols = ScaleUVFilterCols_NEON;
}
}
#endif
// TODO(fbarchard): Consider not allocating row buffer for kFilterLinear.
// Allocate a row of UV.
{
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
uint32_t uy = y;
// clip_src_width is already in bytes (2 per UV pixel).
align_buffer_64(row, clip_src_width);
if (!row)
return 1;
if (uy > max_y) {
uy = max_y;
}
for (j = 0; j < dst_height; ++j) {
int yi = uy >> 16;
const uint8_t* src = src_uv + yi * src_stride;
if (filtering == kFilterLinear) {
ScaleUVFilterCols(dst_uv, src, dst_width, x, dx);
} else {
int yf = (uy >> 8) & 255;
InterpolateRow(row, src, src_stride, clip_src_width, yf);
ScaleUVFilterCols(dst_uv, row, dst_width, x, dx);
}
dst_uv += dst_stride;
uy += dy;
if (uy > max_y) {
uy = max_y;
}
}
free_aligned_buffer_64(row);
}
return 0;
}
#endif
// Scale UV up with bilinear interpolation.
#if HAS_SCALEUVBILINEARUP
static int ScaleUVBilinearUp(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_uv,
uint8_t* dst_uv,
int x,
int dx,
int y,
int dy,
enum FilterMode filtering) {
assert(src_width > 0);
assert(src_height > 0);
assert(dst_width > 0);
assert(dst_height > 0);
assert(dy <= 65536);
int j;
void (*InterpolateRow)(uint8_t* dst_uv, const uint8_t* src_uv,
ptrdiff_t src_stride, int dst_width,
int source_y_fraction) = InterpolateRow_C;
void (*ScaleUVFilterCols)(uint8_t* dst_uv, const uint8_t* src_uv,
int dst_width, int x, int dx) =
filtering ? ScaleUVFilterCols_C : ScaleUVCols_C;
const uint32_t max_y = (uint32_t)(src_height - 1) << 16;
uint32_t uy = y;
#if defined(HAS_INTERPOLATEROW_AVX2)
if (TestCpuFlag(kCpuHasAVX2)) {
InterpolateRow = InterpolateRow_Any_AVX2;
if (IS_ALIGNED(dst_width, 16)) {
InterpolateRow = InterpolateRow_AVX2;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_AVX512BW)
if (TestCpuFlag(kCpuHasAVX512BW)) {
InterpolateRow = InterpolateRow_AVX512BW;
}
#endif
#if defined(HAS_INTERPOLATEROW_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
InterpolateRow = InterpolateRow_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
InterpolateRow = InterpolateRow_NEON;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_SVE2)
if (TestCpuFlag(kCpuHasSVE2)) {
InterpolateRow = InterpolateRow_SVE2;
}
#endif
#if defined(HAS_INTERPOLATEROW_SME)
if (TestCpuFlag(kCpuHasSME)) {
InterpolateRow = InterpolateRow_SME;
}
#endif
#if defined(HAS_INTERPOLATEROW_LSX)
if (TestCpuFlag(kCpuHasLSX)) {
InterpolateRow = InterpolateRow_Any_LSX;
if (IS_ALIGNED(dst_width, 16)) {
InterpolateRow = InterpolateRow_LSX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_LASX)
if (TestCpuFlag(kCpuHasLASX)) {
InterpolateRow = InterpolateRow_Any_LASX;
if (IS_ALIGNED(dst_width, 32)) {
InterpolateRow = InterpolateRow_LASX;
}
}
#endif
#if defined(HAS_INTERPOLATEROW_RVV)
if (TestCpuFlag(kCpuHasRVV)) {
InterpolateRow = InterpolateRow_RVV;
}
#endif
#if defined(HAS_INTERPOLATEROW_HVX)
if (TestCpuFlag(kCpuHasHVX)) {
InterpolateRow = InterpolateRow_HVX;
}
#endif
#if defined(HAS_INTERPOLATEROW_WASMSIMD)
if (TestCpuFlag(kCpuHasWASMSIMD)) {
InterpolateRow = InterpolateRow_Any_WASMSIMD;
if (IS_ALIGNED(dst_width, 8)) {
InterpolateRow = InterpolateRow_WASMSIMD;
}
}
#endif
if (src_width >= 32768) {
ScaleUVFilterCols = filtering ? ScaleUVFilterCols64_C : ScaleUVCols64_C;
}
#if defined(HAS_SCALEUVFILTERCOLS_SSSE3)
if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleUVFilterCols = ScaleUVFilterCols_SSSE3;
}
#endif
#if defined(HAS_SCALEUVFILTERCOLS_NEON)
if (filtering && TestCpuFlag(kCpuHasNEON)) {
ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVFilterCols = ScaleUVFilterCols_NEON;
}
}
#endif
#if defined(HAS_SCALEUVCOLS_SSSE3)
if (!filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleUVFilterCols = ScaleUVCols_SSSE3;
}
#endif
#if defined(HAS_SCALEUVCOLS_NEON)
if (!filtering && TestCpuFlag(kCpuHasNEON)) {
ScaleUVFilterCols = ScaleUVCols_Any_NEON;
if (IS_ALIGNED(dst_width, 16)) {
ScaleUVFilterCols = ScaleUVCols_NEON;
}
}
#endif
if (!filtering && src_width * 2 == dst_width && x < 0x8000) {
ScaleUVFilterCols = ScaleUVColsUp2_C;
#if defined(HAS_SCALEUVCOLSUP2_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) {
ScaleUVFilterCols = ScaleUVColsUp2_SSSE3;
}
#endif
}
if (uy > max_y) {
uy = max_y;
}
{
int yi = uy >> 16;
const uint8_t* src = src_uv + yi * src_stride;
// Allocate 2 rows of UV.
const int row_size = (dst_width * 2 + 15) & ~15;
align_buffer_64(row, row_size * 2);
if (!row)
return 1;
uint8_t* rowptr = row;
ptrdiff_t rowstride = row_size;
int lasty = yi;
ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
if (src_height > 1) {
src += src_stride;
}
ScaleUVFilterCols(rowptr + rowstride, src, dst_width, x, dx);
if (src_height > 2) {
src += src_stride;
}
// 2-row rolling buffer:
// rowptr and (rowptr + rowstride) hold the scaled rows for yi and yi + 1.
// Because dy <= 65536 (dy <= 1.0 in 16.16), yi advances in unit steps.
// When yi != lasty:
// 1. Scale the next source row into the older buffer (rowptr).
// 2. Swap buffer pointers (rowptr += rowstride; rowstride = -rowstride;)
// so rowptr points to yi and (rowptr + rowstride) points to yi + 1.
// 3. Advance src by 1 row if row yi + 2 exists ((uy + 65536) < max_y),
// otherwise clamp src at (src_height - 1) to avoid reading out of
// bounds.
for (j = 0; j < dst_height; ++j) {
if (uy > max_y) {
uy = max_y;
}
yi = uy >> 16;
if (yi != lasty) {
ScaleUVFilterCols(rowptr, src, dst_width, x, dx);
rowptr += rowstride;
rowstride = -rowstride;
lasty = yi;
if ((uy + 65536) < max_y) {
src += src_stride;
}
}
if (filtering == kFilterLinear) {
InterpolateRow(dst_uv, rowptr, 0, dst_width * 2, 0);
} else {
int yf = (uy >> 8) & 255;
InterpolateRow(dst_uv, rowptr, rowstride, dst_width * 2, yf);
}
dst_uv += dst_stride;
uy += dy;
}
free_aligned_buffer_64(row);
}
return 0;
}
#endif // HAS_SCALEUVBILINEARUP
// Scale UV, horizontally up by 2 times.
// Uses linear filter horizontally, nearest vertically.
// This is an optimized version for scaling up a plane to 2 times of
// its original width, using linear interpolation.
// This is used to scale U and V planes of NV16 to NV24.
static void ScaleUVLinearUp2(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_uv,
uint8_t* dst_uv) {
void (*ScaleRowUp)(const uint8_t* src_uv, uint8_t* dst_uv, int dst_width) =
ScaleUVRowUp2_Linear_Any_C;
int i;
// This function can only scale up by 2 times horizontally.
(void)src_width;
assert(src_width == ((dst_width + 1) / 2));
#ifdef HAS_SCALEUVROWUP2_LINEAR_SSSE3
if (TestCpuFlag(kCpuHasSSSE3)) {
ScaleRowUp = ScaleUVRowUp2_Linear_Any_SSSE3;
}
#endif
#ifdef HAS_SCALEUVROWUP2_LINEAR_AVX2
if (TestCpuFlag(kCpuHasAVX2)) {
ScaleRowUp = ScaleUVRowUp2_Linear_Any_AVX2;
}
#endif
#ifdef HAS_SCALEUVROWUP2_LINEAR_NEON
if (TestCpuFlag(kCpuHasNEON)) {
ScaleRowUp = ScaleUVRowUp2_Linear_Any_NEON;
}
#endif
#ifdef HAS_SCALEUVROWUP2_LINEAR_RVV
if (TestCpuFlag(kCpuHasRVV)) {
ScaleRowUp = ScaleUVRowUp2_Linear_RVV;
}
#endif
if (dst_height == 1) {
ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width);
} else {
uint32_t uy = (1 << 15) - 1;
int dy = FixedDiv(src_height - 1, dst_height - 1);
for (i = 0; i < dst_height; ++i) {
int yi = uy >> 16;
ScaleRowUp(src_uv + yi * src_stride, dst_uv, dst_width);
dst_uv += dst_stride;
uy += dy;
}
}
}
// Scale plane, up by 2 times.
// This is an optimized version for scaling up a plane to 2 times of
// its original size, using bilinear interpolation.
// This is used to scale U and V planes of NV12 to NV24.
static void ScaleUVBilinearUp2(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_ptr,
uint8_t* dst_ptr) {
void (*Scale2RowUp)(const uint8_t* src_ptr, ptrdiff_t src_stride,
uint8_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) =
ScaleUVRowUp2_Bilinear_Any_C;
int x;
// This function can only scale up by 2 times.
(void)src_width;
assert(src_width == ((dst_width + 1) / 2));
assert(src_height == ((dst_height + 1) / 2));
#ifdef HAS_SCALEUVROWUP2_BILINEAR_SSSE3
if (TestCpuFlag(kCpuHasSSSE3)) {
Scale2RowUp = ScaleUVRowUp2_Bilinear_Any_SSSE3;
}
#endif
#ifdef HAS_SCALEUVROWUP2_BILINEAR_AVX2
if (TestCpuFlag(kCpuHasAVX2)) {
Scale2RowUp = ScaleUVRowUp2_Bilinear_Any_AVX2;
}
#endif
#ifdef HAS_SCALEUVROWUP2_BILINEAR_NEON
if (TestCpuFlag(kCpuHasNEON)) {
Scale2RowUp = ScaleUVRowUp2_Bilinear_Any_NEON;
}
#endif
#ifdef HAS_SCALEUVROWUP2_BILINEAR_RVV
if (TestCpuFlag(kCpuHasRVV)) {
Scale2RowUp = ScaleUVRowUp2_Bilinear_RVV;
}
#endif
Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
dst_ptr += dst_stride;
for (x = 0; x < src_height - 1; ++x) {
Scale2RowUp(src_ptr, src_stride, dst_ptr, dst_stride, dst_width);
src_ptr += src_stride;
// TODO(fbarchard): Test performance of writing one row of destination at a
// time.
dst_ptr += 2 * dst_stride;
}
if (!(dst_height & 1)) {
Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
}
}
// Scale 16 bit UV, horizontally up by 2 times.
// Uses linear filter horizontally, nearest vertically.
// This is an optimized version for scaling up a plane to 2 times of
// its original width, using linear interpolation.
// This is used to scale U and V planes of P210 to P410.
static void ScaleUVLinearUp2_16(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint16_t* src_uv,
uint16_t* dst_uv) {
void (*ScaleRowUp)(const uint16_t* src_uv, uint16_t* dst_uv, int dst_width) =
ScaleUVRowUp2_Linear_16_Any_C;
int i;
// This function can only scale up by 2 times horizontally.
(void)src_width;
assert(src_width == ((dst_width + 1) / 2));
#ifdef HAS_SCALEUVROWUP2_LINEAR_16_SSE41
if (TestCpuFlag(kCpuHasSSE41)) {
ScaleRowUp = ScaleUVRowUp2_Linear_16_Any_SSE41;
}
#endif
#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2
if (TestCpuFlag(kCpuHasAVX2)) {
ScaleRowUp = ScaleUVRowUp2_Linear_16_Any_AVX2;
}
#endif
#ifdef HAS_SCALEUVROWUP2_LINEAR_16_NEON
if (TestCpuFlag(kCpuHasNEON)) {
ScaleRowUp = ScaleUVRowUp2_Linear_16_Any_NEON;
}
#endif
if (dst_height == 1) {
ScaleRowUp(src_uv + ((src_height - 1) / 2) * src_stride, dst_uv, dst_width);
} else {
uint32_t uy = (1 << 15) - 1;
int dy = FixedDiv(src_height - 1, dst_height - 1);
for (i = 0; i < dst_height; ++i) {
int yi = uy >> 16;
ScaleRowUp(src_uv + yi * src_stride, dst_uv, dst_width);
dst_uv += dst_stride;
uy += dy;
}
}
}
// Scale 16 bit UV, up by 2 times.
// This is an optimized version for scaling up a plane to 2 times of
// its original size, using bilinear interpolation.
// This is used to scale U and V planes of P010 to P410.
static void ScaleUVBilinearUp2_16(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint16_t* src_ptr,
uint16_t* dst_ptr) {
void (*Scale2RowUp)(const uint16_t* src_ptr, ptrdiff_t src_stride,
uint16_t* dst_ptr, ptrdiff_t dst_stride, int dst_width) =
ScaleUVRowUp2_Bilinear_16_Any_C;
int x;
// This function can only scale up by 2 times.
(void)src_width;
assert(src_width == ((dst_width + 1) / 2));
assert(src_height == ((dst_height + 1) / 2));
#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_SSE41
if (TestCpuFlag(kCpuHasSSE41)) {
Scale2RowUp = ScaleUVRowUp2_Bilinear_16_Any_SSE41;
}
#endif
#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2
if (TestCpuFlag(kCpuHasAVX2)) {
Scale2RowUp = ScaleUVRowUp2_Bilinear_16_Any_AVX2;
}
#endif
#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_NEON
if (TestCpuFlag(kCpuHasNEON)) {
Scale2RowUp = ScaleUVRowUp2_Bilinear_16_Any_NEON;
}
#endif
Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
dst_ptr += dst_stride;
for (x = 0; x < src_height - 1; ++x) {
Scale2RowUp(src_ptr, src_stride, dst_ptr, dst_stride, dst_width);
src_ptr += src_stride;
// TODO(fbarchard): Test performance of writing one row of destination at a
// time.
dst_ptr += 2 * dst_stride;
}
if (!(dst_height & 1)) {
Scale2RowUp(src_ptr, 0, dst_ptr, 0, dst_width);
}
}
// Scale UV to/from any dimensions, without interpolation.
// Fixed point math is used for performance: The upper 16 bits
// of x and dx is the integer part of the source position and
// the lower 16 bits are the fixed decimal part.
static void ScaleUVSimple(int src_width,
int src_height,
int dst_width,
int dst_height,
ptrdiff_t src_stride,
ptrdiff_t dst_stride,
const uint8_t* src_uv,
uint8_t* dst_uv,
int x,
int dx,
int y,
int dy) {
int j;
void (*ScaleUVCols)(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width,
int x, int dx) =
(src_width >= 32768) ? ScaleUVCols64_C : ScaleUVCols_C;
(void)src_height;
#if defined(HAS_SCALEUVCOLS_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) {
ScaleUVCols = ScaleUVCols_SSSE3;
}
#endif
#if defined(HAS_SCALEUVCOLS_NEON)
if (TestCpuFlag(kCpuHasNEON)) {
ScaleUVCols = ScaleUVCols_Any_NEON;
if (IS_ALIGNED(dst_width, 8)) {
ScaleUVCols = ScaleUVCols_NEON;
}
}
#endif
if (src_width * 2 == dst_width && x < 0x8000) {
ScaleUVCols = ScaleUVColsUp2_C;
#if defined(HAS_SCALEUVCOLSUP2_SSSE3)
if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) {
ScaleUVCols = ScaleUVColsUp2_SSSE3;
}
#endif
}
uint32_t uy = y;
for (j = 0; j < dst_height; ++j) {
int yi = uy >> 16;
ScaleUVCols(dst_uv, src_uv + yi * src_stride, dst_width, x, dx);
dst_uv += dst_stride;
uy += dy;
}
}
// Copy UV with optional flipping
#if HAS_UVCOPY
static int UVCopy(const uint8_t* src_uv,
int src_stride_uv,
uint8_t* dst_uv,
int dst_stride_uv,
int width,
int height) {
if (!src_uv || !dst_uv || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
src_uv = src_uv + (height - 1) * (ptrdiff_t)src_stride_uv;
src_stride_uv = -src_stride_uv;
}
CopyPlane(src_uv, src_stride_uv, dst_uv, dst_stride_uv, width * 2, height);
return 0;
}
static int UVCopy_16(const uint16_t* src_uv,
int src_stride_uv,
uint16_t* dst_uv,
int dst_stride_uv,
int width,
int height) {
if (!src_uv || !dst_uv || width <= 0 || height == 0 || height == INT_MIN) {
return -1;
}
// Negative height means invert the image.
if (height < 0) {
height = -height;
src_uv = src_uv + (height - 1) * (ptrdiff_t)src_stride_uv;
src_stride_uv = -src_stride_uv;
}
CopyPlane_16(src_uv, src_stride_uv, dst_uv, dst_stride_uv, width * 2, height);
return 0;
}
#endif // HAS_UVCOPY
// Scale a UV plane (from NV12)
// This function in turn calls a scaling function
// suitable for handling the desired resolutions.
static int ScaleUV(const uint8_t* src,
int src_stride,
int src_width,
int src_height,
uint8_t* dst,
int dst_stride,
int dst_width,
int dst_height,
int clip_x,
int clip_y,
int clip_width,
int clip_height,
enum FilterMode filtering) {
// Initial source x/y coordinate and step values as 16.16 fixed point.
int x = 0;
int y = 0;
int dx = 0;
int dy = 0;
// Simplify filtering when possible.
filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
filtering);
// Negative src_height means invert the image.
if (src_height < 0) {
src_height = -src_height;
src = src + (src_height - 1) * (ptrdiff_t)src_stride;
src_stride = -src_stride;
}
ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y,
&dx, &dy);
if (clip_x) {
int64_t clipf = (int64_t)(clip_x)*dx;
x += (clipf & 0xffff);
src += (clipf >> 16) * 2;
dst += clip_x * 2;
}
if (clip_y) {
int64_t clipf = (int64_t)(clip_y)*dy;
y += (clipf & 0xffff);
src += (clipf >> 16) * (ptrdiff_t)src_stride;
dst += clip_y * (ptrdiff_t)dst_stride;
}
#if HAS_SCALEUVBOX
// Exactly 4 goes to the faster ScaleUVDown4Box below. Clipping is excluded
// because whole source rows are accumulated.
if (filtering == kFilterBox && !(dx == 0x40000 && dy == 0x40000) &&
clip_x == 0 && clip_y == 0) {
return ScaleUVBox(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy);
}
#endif
// Special case for integer step values.
if (((dx | dy) & 0xffff) == 0) {
if (!dx || !dy) { // 1 pixel wide and/or tall.
filtering = kFilterNone;
} else {
// Optimized even scale down. ie 2, 4, 6, 8, 10x.
if (!(dx & 0x10000) && !(dy & 0x10000)) {
#if HAS_SCALEUVDOWN2
if (dx == 0x20000 && dy == 0x20000) {
// Optimized 1/2 downsample.
ScaleUVDown2(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy,
filtering);
return 0;
}
#endif
#if HAS_SCALEUVDOWN4BOX
if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) {
// Optimized 1/4 box downsample.
return ScaleUVDown4Box(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y,
dy);
}
#endif
#if HAS_SCALEUVDOWNEVEN
ScaleUVDownEven(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy,
filtering);
return 0;
#endif
}
// Optimized odd scale down. ie 3, 5, 7, 9x.
if ((dx & 0x10000) && (dy & 0x10000)) {
filtering = kFilterNone;
#ifdef HAS_UVCOPY
if (dx == 0x10000 && dy == 0x10000) {
// Straight copy.
return UVCopy(src + (y >> 16) * (ptrdiff_t)src_stride + (x >> 16) * 2,
src_stride, dst, dst_stride, clip_width, clip_height);
}
#endif
}
}
}
// HAS_SCALEPLANEVERTICAL
if (dx == 0x10000 && (x & 0xffff) == 0) {
// Arbitrary scale vertically, but unscaled horizontally.
ScalePlaneVertical(src_height, clip_width, clip_height, src_stride,
dst_stride, src, dst, x, y, dy, /*bpp=*/2, filtering);
return 0;
}
if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
ScaleUVLinearUp2(src_width, src_height, clip_width, clip_height, src_stride,
dst_stride, src, dst);
return 0;
}
if ((clip_height + 1) / 2 == src_height &&
(clip_width + 1) / 2 == src_width &&
(filtering == kFilterBilinear || filtering == kFilterBox)) {
ScaleUVBilinearUp2(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst);
return 0;
}
#if HAS_SCALEUVBILINEARUP
if (filtering && dy < 65536) {
return ScaleUVBilinearUp(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy,
filtering);
}
#endif
#if HAS_SCALEUVBILINEARDOWN
if (filtering) {
return ScaleUVBilinearDown(src_width, src_height, clip_width, clip_height,
src_stride, dst_stride, src, dst, x, dx, y, dy,
filtering);
}
#endif
ScaleUVSimple(src_width, src_height, clip_width, clip_height, src_stride,
dst_stride, src, dst, x, dx, y, dy);
return 0;
}
// Scale an UV image.
LIBYUV_API
int UVScale(const uint8_t* src_uv,
int src_stride_uv,
int src_width,
int src_height,
uint8_t* dst_uv,
int dst_stride_uv,
int dst_width,
int dst_height,
enum FilterMode filtering) {
if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 ||
src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 ||
dst_height <= 0) {
return -1;
}
return ScaleUV(src_uv, src_stride_uv, src_width, src_height, dst_uv,
dst_stride_uv, dst_width, dst_height, 0, 0, dst_width,
dst_height, filtering);
}
// Scale a 16 bit UV image.
// This function is currently incomplete, it can't handle all cases.
LIBYUV_API
int UVScale_16(const uint16_t* src_uv,
int src_stride_uv,
int src_width,
int src_height,
uint16_t* dst_uv,
int dst_stride_uv,
int dst_width,
int dst_height,
enum FilterMode filtering) {
int dy = 0;
if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 ||
src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 ||
dst_height <= 0) {
return -1;
}
// UV does not support box filter yet, but allow the user to pass it.
// Simplify filtering when possible.
filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height,
filtering);
// Negative src_height means invert the image.
if (src_height < 0) {
src_height = -src_height;
src_uv = src_uv + (src_height - 1) * (ptrdiff_t)src_stride_uv;
src_stride_uv = -src_stride_uv;
}
#ifdef HAS_UVCOPY
if (!filtering && src_width == dst_width && (src_height % dst_height == 0)) {
if (dst_height == 1) {
return UVCopy_16(
src_uv + ((src_height - 1) / 2) * (ptrdiff_t)src_stride_uv,
src_stride_uv, dst_uv, dst_stride_uv, dst_width, dst_height);
}
dy = src_height / dst_height;
if (src_stride_uv > INT_MAX / dy) {
return -1;
}
return UVCopy_16(src_uv + ((dy - 1) / 2) * (ptrdiff_t)src_stride_uv,
dy * src_stride_uv, dst_uv, dst_stride_uv, dst_width,
dst_height);
}
#endif
// ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone, which has
// no 16 bit implementation other than the copy above. The Up2 kernels only
// read the pixels they need, so restore Linear or Bilinear for a 2x upscale
// so P010ToP410 and P210ToP410 work on 1 pixel wide images.
if (filtering == kFilterNone && src_width == 1 && dst_width <= 2) {
filtering = (src_height == dst_height) ? kFilterLinear : kFilterBilinear;
}
if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) {
ScaleUVLinearUp2_16(src_width, src_height, dst_width, dst_height,
src_stride_uv, dst_stride_uv, src_uv, dst_uv);
return 0;
}
if ((dst_height + 1) / 2 == src_height && (dst_width + 1) / 2 == src_width &&
(filtering == kFilterBilinear || filtering == kFilterBox)) {
ScaleUVBilinearUp2_16(src_width, src_height, dst_width, dst_height,
src_stride_uv, dst_stride_uv, src_uv, dst_uv);
return 0;
}
return -1;
}
#ifdef __cplusplus
} // extern "C"
} // namespace libyuv
#endif
Reference in New Issue View Git Blame Copy Permalink