diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 8ea8e0100..88a975dd4 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -35,6 +35,9 @@ extern "C" { #ifndef HAS_SCALEUVDOWNEVEN #define HAS_SCALEUVDOWNEVEN 1 #endif +#ifndef HAS_SCALEUVBOX +#define HAS_SCALEUVBOX 1 +#endif #ifndef HAS_SCALEUVBILINEARDOWN #define HAS_SCALEUVBILINEARDOWN 1 #endif @@ -52,6 +55,8 @@ static __inline int Abs(int v) { return v >= 0 ? v : -v; } +#define MIN1(x) ((x) < 1 ? 1 : (x)) + // ScaleUV, 1/2 // This is an optimized version for scaling down a UV to 1/2 of // its original size. @@ -249,6 +254,154 @@ static int ScaleUVDown4Box(int src_width, } #endif // HAS_SCALEUVDOWN4BOX +// ScaleUV, any dimensions, box filter. +// The counterpart of ScalePlaneBox in scale.cc. Rows are summed into 16 bit +// accumulators, then each destination pixel averages its box of source pixels. +#if HAS_SCALEUVBOX +static void ScaleUVAddCols1_C(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_uv, + uint8_t* dst_uv) { + int i, j; + int boxwidth = MIN1(dx >> 16); + int scaleval = 65536 / (boxwidth * boxheight); + x >>= 16; + for (i = 0; i < dst_width; ++i) { + int u = 0; + int v = 0; + for (j = 0; j < boxwidth; ++j) { + u += src_uv[(x + j) * 2]; + v += src_uv[(x + j) * 2 + 1]; + } + x += boxwidth; + *dst_uv++ = (uint8_t)(u * scaleval >> 16); + *dst_uv++ = (uint8_t)(v * scaleval >> 16); + } +} + +static void ScaleUVAddCols2_C(int dst_width, + int boxheight, + int x, + int dx, + const uint16_t* src_uv, + uint8_t* dst_uv) { + int i, j; + int minboxwidth = dx >> 16; + int scaletbl[2]; + scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight); + scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight); + for (i = 0; i < dst_width; ++i) { + int ix = x >> 16; + int u = 0; + int v = 0; + int boxwidth; + int scaleval; + x += dx; + boxwidth = MIN1((x >> 16) - ix); + assert((boxwidth - minboxwidth == 0) || (boxwidth - minboxwidth == 1)); + scaleval = scaletbl[boxwidth - minboxwidth]; + for (j = 0; j < boxwidth; ++j) { + u += src_uv[(ix + j) * 2]; + v += src_uv[(ix + j) * 2 + 1]; + } + *dst_uv++ = (uint8_t)(u * scaleval >> 16); + *dst_uv++ = (uint8_t)(v * scaleval >> 16); + } +} + +static int ScaleUVBox(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint8_t* src_uv, + uint8_t* dst_uv, + int x, + int dx, + int y, + int dy) { + int j, k; + const int max_y = (src_height << 16); + // Sized on src_width to keep ScaleAddRow aligned. + const int row_samples = src_width * 2; + { + // Row accumulator, two uint16_t per source pixel. + align_buffer_64(row16, row_samples * 2); + if (!row16) + return 1; + { + void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx, + const uint16_t* src_uv, uint8_t* dst_uv) = + (dx & 0xffff) ? ScaleUVAddCols2_C : ScaleUVAddCols1_C; + // Summing a row is position wise, so an interleaved UV row can be fed to + // the plane accumulator as src_width * 2 samples. + void (*ScaleAddRow)(const uint8_t* src_ptr, uint16_t* dst_ptr, + int src_width) = ScaleAddRow_C; +#if defined(HAS_SCALEADDROW_SSE2) + if (TestCpuFlag(kCpuHasSSE2)) { + ScaleAddRow = ScaleAddRow_Any_SSE2; + if (IS_ALIGNED(row_samples, 16)) { + ScaleAddRow = ScaleAddRow_SSE2; + } + } +#endif +#if defined(HAS_SCALEADDROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleAddRow = ScaleAddRow_Any_AVX2; + if (IS_ALIGNED(row_samples, 32)) { + ScaleAddRow = ScaleAddRow_AVX2; + } + } +#endif +#if defined(HAS_SCALEADDROW_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + ScaleAddRow = ScaleAddRow_Any_NEON; + if (IS_ALIGNED(row_samples, 16)) { + ScaleAddRow = ScaleAddRow_NEON; + } + } +#endif +#if defined(HAS_SCALEADDROW_LSX) + if (TestCpuFlag(kCpuHasLSX)) { + ScaleAddRow = ScaleAddRow_Any_LSX; + if (IS_ALIGNED(row_samples, 16)) { + ScaleAddRow = ScaleAddRow_LSX; + } + } +#endif +#if defined(HAS_SCALEADDROW_RVV) + if (TestCpuFlag(kCpuHasRVV)) { + ScaleAddRow = ScaleAddRow_RVV; + } +#endif + + for (j = 0; j < dst_height; ++j) { + int boxheight; + int iy = y >> 16; + const uint8_t* src = src_uv + iy * src_stride; + y += dy; + if (y > max_y) { + y = max_y; + } + boxheight = MIN1((y >> 16) - iy); + memset(row16, 0, row_samples * 2); + for (k = 0; k < boxheight; ++k) { + ScaleAddRow(src, (uint16_t*)(row16), row_samples); + src += src_stride; + } + ScaleUVAddCols(dst_width, boxheight, x, dx, (uint16_t*)(row16), dst_uv); + dst_uv += dst_stride; + } + } + free_aligned_buffer_64(row16); + } + return 0; +} +#endif // HAS_SCALEUVBOX + // ScaleUV Even // This is an optimized version for scaling down a UV to even // multiple of its original size. @@ -973,7 +1126,6 @@ static int ScaleUV(const uint8_t* src, int y = 0; int dx = 0; int dy = 0; - // UV does not support box filter yet, but allow the user to pass it. // Simplify filtering when possible. filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height, filtering); @@ -1000,6 +1152,16 @@ static int ScaleUV(const uint8_t* src, dst += clip_y * (ptrdiff_t)dst_stride; } +#if HAS_SCALEUVBOX + // Exactly 4 goes to the faster ScaleUVDown4Box below. Clipping is excluded + // because whole source rows are accumulated. + if (filtering == kFilterBox && !(dx == 0x40000 && dy == 0x40000) && + clip_x == 0 && clip_y == 0) { + return ScaleUVBox(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy); + } +#endif + // Special case for integer step values. if (((dx | dy) & 0xffff) == 0) { if (!dx || !dy) { // 1 pixel wide and/or tall. diff --git a/unit_test/scale_uv_test.cc b/unit_test/scale_uv_test.cc index df1e4c54c..65a7c662f 100644 --- a/unit_test/scale_uv_test.cc +++ b/unit_test/scale_uv_test.cc @@ -13,6 +13,7 @@ #include "../unit_test/unit_test.h" #include "libyuv/cpu_id.h" +#include "libyuv/scale.h" #include "libyuv/scale_uv.h" namespace libyuv { @@ -246,4 +247,122 @@ TEST_F(LibYUVScaleTest, UVTest4x) { free_aligned_buffer_page_end(orig_pixels); } +// Box filtering an interleaved UV plane has to produce the same bytes as box +// filtering the U and V planes separately, which is the path I420 takes. +// C and SIMD agreeing is not enough here: before ScaleUVBox existed both +// point sampled, so this compares against an independent implementation. +static int UVTestBoxAgainstPlanes(int src_width, + int src_height, + int dst_width, + int dst_height, + int cpu_info) { + if (!SizeValid(src_width, src_height, dst_width, dst_height)) { + return 0; + } + int64_t src_uv_plane_size = src_width * src_height * 2LL; + int64_t src_plane_size = src_width * src_height * 1LL; + int64_t dst_uv_plane_size = dst_width * dst_height * 2LL; + int64_t dst_plane_size = dst_width * dst_height * 1LL; + + align_buffer_page_end(src_uv, src_uv_plane_size); + align_buffer_page_end(src_u, src_plane_size); + align_buffer_page_end(src_v, src_plane_size); + align_buffer_page_end(dst_uv, dst_uv_plane_size); + align_buffer_page_end(dst_u, dst_plane_size); + align_buffer_page_end(dst_v, dst_plane_size); + if (!src_uv || !src_u || !src_v || !dst_uv || !dst_u || !dst_v) { + printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n"); + return 0; + } + MemRandomize(src_uv, src_uv_plane_size); + memset(dst_uv, 123, dst_uv_plane_size); + for (int i = 0; i < src_width * src_height; ++i) { + src_u[i] = src_uv[i * 2 + 0]; + src_v[i] = src_uv[i * 2 + 1]; + } + + MaskCpuFlags(cpu_info); + UVScale(src_uv, src_width * 2, src_width, src_height, dst_uv, dst_width * 2, + dst_width, dst_height, kFilterBox); + ScalePlane(src_u, src_width, src_width, src_height, dst_u, dst_width, + dst_width, dst_height, kFilterBox); + ScalePlane(src_v, src_width, src_width, src_height, dst_v, dst_width, + dst_width, dst_height, kFilterBox); + + int max_diff = 0; + for (int i = 0; i < dst_width * dst_height; ++i) { + int du = Abs(dst_uv[i * 2 + 0] - dst_u[i]); + int dv = Abs(dst_uv[i * 2 + 1] - dst_v[i]); + if (du > max_diff) { + max_diff = du; + } + if (dv > max_diff) { + max_diff = dv; + } + } + + free_aligned_buffer_page_end(dst_v); + free_aligned_buffer_page_end(dst_u); + free_aligned_buffer_page_end(dst_uv); + free_aligned_buffer_page_end(src_v); + free_aligned_buffer_page_end(src_u); + free_aligned_buffer_page_end(src_uv); + return max_diff; +} + +// Chroma dimensions, ie half the frame sizes named in the comments. +// max_diff is 0 for everything ScaleUVBox handles. The 1/2 and 1/4 fast paths +// predate it and keep their own rounding, so 1/4 is allowed to drift by one. +#define TEST_BOXPLANE(name, swidth, sheight, dwidth, dheight, max_diff) \ + TEST_F(LibYUVScaleTest, UVScaleBoxMatchesPlane##name) { \ + int diff_c = UVTestBoxAgainstPlanes(swidth, sheight, dwidth, dheight, \ + disable_cpu_flags_); \ + ASSERT_LE(diff_c, max_diff); \ + int diff_opt = UVTestBoxAgainstPlanes(swidth, sheight, dwidth, dheight, \ + benchmark_cpu_info_); \ + ASSERT_LE(diff_opt, max_diff); \ + } + +TEST_BOXPLANE(By2, 960, 540, 480, 270, 0) /* 1080p to 960x540 */ +TEST_BOXPLANE(By3, 960, 540, 320, 180, 0) /* 1080p to 640x360 */ +TEST_BOXPLANE(By3Rounded, 640, 360, 214, 120, 0) /* 720p to 427x240 */ +TEST_BOXPLANE(By4, 960, 540, 240, 135, 1) /* ScaleUVDown4Box */ +TEST_BOXPLANE(By3_75, 960, 540, 256, 144, 0) /* 1080p to 512x288 */ +TEST_BOXPLANE(By6, 640, 360, 106, 60, 0) /* 720p to 213x120 */ +TEST_BOXPLANE(By8, 640, 360, 80, 45, 0) /* 720p to 160x90 */ +TEST_BOXPLANE(OddTail, 33, 17, 7, 5, 0) /* not a multiple of 8 wide */ +TEST_BOXPLANE(Tiny, 80, 60, 3, 2, 0) /* dst below one SIMD group */ +#undef TEST_BOXPLANE + +// A box filter has to read the whole box. Every 3x3 block here is 255 except +// for its top left pixel, so point sampling the block origin yields 0 while +// the box average is 8 * 255 * (65536 / 9) >> 16 = 226. +TEST_F(LibYUVScaleTest, UVScaleBoxAverages) { + const int kSrcWidth = 48; + const int kSrcHeight = 24; + const int kDstWidth = kSrcWidth / 3; + const int kDstHeight = kSrcHeight / 3; + align_buffer_page_end(src_uv, kSrcWidth * kSrcHeight * 2); + align_buffer_page_end(dst_uv, kDstWidth * kDstHeight * 2); + + memset(src_uv, 255, kSrcWidth * kSrcHeight * 2); + for (int y = 0; y < kSrcHeight; y += 3) { + for (int x = 0; x < kSrcWidth; x += 3) { + src_uv[(y * kSrcWidth + x) * 2 + 0] = 0; + src_uv[(y * kSrcWidth + x) * 2 + 1] = 0; + } + } + memset(dst_uv, 123, kDstWidth * kDstHeight * 2); + + UVScale(src_uv, kSrcWidth * 2, kSrcWidth, kSrcHeight, dst_uv, kDstWidth * 2, + kDstWidth, kDstHeight, kFilterBox); + + for (int i = 0; i < kDstWidth * kDstHeight * 2; ++i) { + ASSERT_EQ(226, dst_uv[i]) << "at " << i; + } + + free_aligned_buffer_page_end(dst_uv); + free_aligned_buffer_page_end(src_uv); +} + } // namespace libyuv