From 8d440c181c1bdc884c2d2d6d23a773ae51d178a1 Mon Sep 17 00:00:00 2001 From: Frank Barchard Date: Wed, 28 Oct 2026 13:27:48 -0700 Subject: [PATCH] [libyuv] Upgrade x86 SIMD to AVX2/AVX-512 with 32-bit & row_win parity Upgrade all remaining SSE2 and SSSE3 max-ISA kernels across row, scale, rotate, and compare to AVX2 while removing superseded pre-AVX2 versions. Implement AVX2 for ARGBDetectRow, HalfRow, HalfRow_16, HalfWidthRow_16To8_Odd, AYUVTo{Y,UV,VU}Row, AR30To{ARGB,ABGR,AB30}Row, and TransposeWx8_16, and port all remaining Arm NEON (or higher) kernels to AVX2 (TransposeWxH_32, ScaleRowDown2*_16, ScaleUVRowDown2*, ScaleUVRowDownEven, ScaleColsUp2, GaussCol*, Scale*Samples, and UnpackMT2T). Port 32-bit x86 (-m32 -fPIC) inline assembly across row_gcc.cc, rotate_gcc.cc, and compare_gcc.cc (including 5-GPR PIC support for I422ToAR30Row and I444/I422/I210/I410AlphaToARGBRow) so x86_32 matches x64 SIMD coverage across all 187 supported row/rotate/compare kernels. Handle 1..7 remainder elements directly in inline assembly without calling *_C in ARGBDetectRow_AVX2, ARGBSepiaRow_AVX2, ARGBColorMatrixRow_AVX2, ARGBQuantizeRow_AVX2, ARGBShadeRow_AVX2, ARGBAffineRow_AVX2, GaussRow*_AVX2, GaussCol*_AVX2, and *ColorTableRow_AVX2. Fix unsigned 16-bit packing in ScaleRowUp2_{Linear,Bilinear}_16_SSE2. Add 64-bit AVX-512 (AVX512BW, AVX512VBMI, and AVX512VNNI, including SumSquareError_AVX512VNNI) implementations in *_gcc.cc and *_win.cc wherever Arm64 exceeds NEON, and handle arbitrary tail widths via opmasks in Mirror*_AVX512BW without _Any_ wrappers. Bring row_win.cc, scale_win.cc, rotate_win.cc, and compare_win.cc C++ intrinsics (and 64-bit VISUALC_HAS_AVX2) to 100% parity with *_gcc.cc for MSVC and MemorySanitizer. Wire YUY2ToUVRow_HVX and UYVYToUVRow_HVX in YUY2ToI420 and UYVYToI420, implement all 13 missing ScaleUV*_16_C kernels so UVScale_16 supports downscaling and general upscaling, make Ssim8x8, TransposeWxH, TransposeUVWxH, and TransposeWxH_16 static internal helpers, clean up HAS_* guards, enable direct row unittests so all 279 *_C functions have active test coverage, and add Intel AMX-TF32 (kCpuHasAMXTF32, CPUID leaf 7 subleaf 1 EAX bit 4, LIBYUV_DISABLE_AMXTF32) detection. Mark 16-bit RGB formats (RGB565, ARGB1555, and ARGB4444) and legacy fixed-matrix 16-bit RGB-to-YUV row functions as deprecated in headers and GEMINI.md, using two-step ARGB wrapper row functions in row_common.cc. AMD Zen 6 Benchmark (1280 width, 10000 iterations): ---------------------------------------------------------------- Function Before (ms) After (ms) Improvement ---------------------------------------------------------------- ARGB1555ToARGBRow 0.83 0.49 1.69x ARGB1555ToUVRow 16.00 3.21 4.99x ARGB1555ToYRow 7.82 1.38 5.66x ARGB4444ToUVRow 15.04 2.58 5.83x ARGB4444ToYRow 7.42 1.07 6.94x ARGBAffineRow 3.51 2.75 1.28x ARGBColorMatrixRow 4.09 1.13 3.63x ARGBColorTableRow 11.21 8.93 1.26x ARGBLumaColorTableRow 10.01 10.21 0.98x ARGBQuantizeRow 2.23 1.21 1.84x ARGBSepiaRow 3.29 0.81 4.04x ARGBSetRow 0.21 0.21 1.02x ARGBShadeRow 1.66 0.87 1.91x ARGBToRGB565DitherRow 1.22 0.50 2.43x ARGBToRGB565Row 1.09 0.44 2.47x ByteToFloatRow 0.88 0.41 2.14x Convert8To8Row 0.21 0.11 1.88x CopyRow_16 0.11 0.11 0.97x DetileRow 0.21 0.12 1.75x DetileRow_16 0.23 0.14 1.67x DetileSplitUVRow 0.29 0.21 1.38x DivideRow_16 0.14 0.11 1.30x GaussRow 3.29 1.27 2.59x GaussRow_F32 1.93 0.89 2.17x HalfFloatRow 1.16 0.74 1.57x I210AlphaToARGBRow 1.83 0.92 2.00x I210ToAR30Row 2.21 0.96 2.30x I210ToARGBRow 1.75 0.87 2.00x I212ToAR30Row 2.21 0.96 2.31x I212ToARGBRow 1.75 0.87 2.01x I410AlphaToARGBRow 1.73 0.86 2.00x I410ToAR30Row 2.04 0.89 2.28x I410ToARGBRow 1.64 0.82 2.00x I422ToARGB1555Row 2.94 2.27 1.30x I422ToARGB4444Row 2.94 2.27 1.29x I422ToRGB565Row 2.63 1.27 2.07x I422ToRGBARow 1.51 0.82 1.83x I444AlphaToARGBRow 1.52 0.87 1.73x I444ToARGBRow 1.48 0.81 1.82x I444ToRGB24Row 2.13 0.72 2.98x MergeUVRow_16 0.87 0.32 2.74x P210ToAR30Row 1.77 0.70 2.54x P210ToARGBRow 1.38 0.62 2.24x P410ToAR30Row 1.94 0.85 2.28x P410ToARGBRow 1.45 0.77 1.90x RGB565ToUVRow 15.84 2.88 5.49x RGB565ToYRow 7.82 1.22 6.42x RGBColorTableRow 8.42 6.95 1.21x SetRow 0.11 0.11 1.04x SobelRow 0.80 0.43 1.87x SobelToPlaneRow 0.27 0.12 2.25x SobelXRow 1.58 0.85 1.85x SobelXYRow 0.80 0.55 1.44x SobelYRow 1.58 0.85 1.85x UYVYToARGBRow 1.28 0.65 1.96x UYVYToUVRow 0.38 0.21 1.78x UYVYToYRow 0.21 0.11 1.87x YUY2ToARGBRow 1.28 0.65 1.96x YUY2ToUVRow 0.40 0.21 1.87x YUY2ToYRow 0.21 0.11 1.88x ---------------------------------------------------------------- Row/Rotate/Compare SIMD ISA per *_C (* = uncalled in library): ------------------------------------------------------------------------ C Function arm64 x86_32 x64 row_win ------------------------------------------------------------------------ AB64ToARGBRow_C NEON AVX2 AVX2 AVX2 ABGRToAR30Row_C NEON AVX2 AVX512BW AVX512BW AR30ToAB30Row_C NEON AVX2 AVX512BW AVX512BW AR30ToABGRRow_C NEON AVX2 AVX512BW AVX512BW AR30ToARGBRow_C NEON AVX2 AVX512BW AVX512BW AR64ShuffleRow_C NEON AVX2 AVX512BW AVX512BW AR64ToAB64Row_C NEON AVX2 AVX512BW AVX512BW AR64ToARGBRow_C NEON AVX2 AVX2 AVX2 ARGB1555ToARGBRow_C SVE2 AVX2 AVX512BW AVX512BW ARGB1555ToUVMatrixRow_C NEON AVX2 AVX2 AVX2 ARGB1555ToUVRow_C * NEON AVX2 AVX2 AVX2 ARGB1555ToYMatrixRow_C NEON AVX2 AVX2 AVX2 ARGB1555ToYRow_C * NEON AVX2 AVX2 AVX2 ARGB4444ToARGBRow_C NEON AVX2 AVX2 AVX2 ARGB4444ToUVMatrixRow_C NEON AVX2 AVX2 AVX2 ARGB4444ToUVRow_C * NEON AVX2 AVX2 AVX2 ARGB4444ToYMatrixRow_C NEON AVX2 AVX2 AVX2 ARGB4444ToYRow_C * NEON AVX2 AVX2 AVX2 ARGBAddRow_C NEON AVX2 AVX2 AVX2 ARGBAffineRow_C * NEON AVX2 AVX2 AVX2 ARGBAlphaBlendRow_C SME AVX2 AVX512BW AVX512BW ARGBAttenuateRow_C NEON AVX2 AVX512BW AVX512BW ARGBBlendRow_C NEON AVX2 AVX512BW AVX512BW ARGBColorMatrixRow_C I8MM AVX2 AVX512VNNI AVX512VNNI ARGBColorTableRow_C NEON AVX2 AVX2 AVX2 ARGBCopyAlphaRow_C NEON AVX2 AVX2 AVX2 ARGBCopyYToAlphaRow_C NEON AVX2 AVX512BW AVX512BW ARGBDetectRow_C - AVX2 AVX2 AVX2 ARGBExtractAlphaRow_C NEON AVX2 AVX512BW AVX512BW ARGBGrayRow_C DOTPROD AVX2 AVX512VNNI AVX512VNNI ARGBLumaColorTableRow_C NEON AVX2 AVX2 AVX2 ARGBMirrorRow_C NEON AVX2 AVX512BW AVX512BW ARGBMultiplyRow_C SME AVX2 AVX512BW AVX512BW ARGBOrRow_C SME AVX2 AVX512BW AVX512BW ARGBPolynomialRow_C NEON AVX2 AVX2 AVX2 ARGBQuantizeRow_C NEON AVX2 AVX2 AVX2 ARGBSepiaRow_C DOTPROD AVX2 AVX512VNNI AVX512VNNI ARGBSetRow_C NEON AVX2 AVX2 AVX2 ARGBShadeRow_C NEON AVX2 AVX2 AVX2 ARGBShuffleRow_C NEON AVX2 AVX512BW AVX512BW ARGBSubtractRow_C NEON AVX2 AVX2 AVX2 ARGBToAB64Row_C NEON AVX2 AVX2 AVX2 ARGBToABGRRow_C NEON AVX2 AVX512BW AVX512BW ARGBToAR30Row_C NEON AVX2 AVX512BW AVX512BW ARGBToAR64Row_C NEON AVX2 AVX2 AVX2 ARGBToARGB1555Row_C NEON AVX2 AVX2 AVX2 ARGBToARGB4444Row_C NEON AVX2 AVX2 AVX2 ARGBToBGRARow_C NEON AVX2 AVX512BW AVX512BW ARGBToRAWRow_C SVE2 AVX2 AVX512BW AVX512BW ARGBToRGB24Row_C SVE2 AVX2 AVX512BW AVX512BW ARGBToRGB565DitherRow_C SVE2 AVX2 AVX512BW AVX512BW ARGBToRGB565Row_C SVE2 AVX2 AVX512BW AVX512BW ARGBToRGBARow_C NEON AVX2 AVX512BW AVX512BW ARGBToUV444MatrixRow_C I8MM AVX2 AVX512VNNI AVX512VNNI ARGBToUVMatrixRow_C SME AVX2 AVX512VNNI AVX512VNNI ARGBToYMatrixRow_C SME AVX2 AVX512VNNI AVX512VNNI ARGBUnattenuateRow_C NEON AVX2 AVX2 AVX2 AYUVToUVRow_C SVE2 AVX2 AVX512VBMI AVX512VBMI AYUVToVURow_C SVE2 AVX2 AVX512VBMI AVX512VBMI AYUVToYRow_C NEON AVX2 AVX512VBMI AVX512VBMI BlendPlaneRow_C NEON AVX2 AVX512BW AVX512BW ByteToFloatRow_C NEON AVX2 AVX2 AVX2 ComputeCumulativeSumRow_C NEON AVX2 AVX2 AVX2 Convert16To8Row_C SME AVX2 AVX512BW AVX512BW Convert8To16Row_C SME AVX2 AVX512BW AVX512BW Convert8To8Row_C SME AVX2 AVX512BW AVX512BW CopyRow_16_C NEON AVX2 AVX2 AVX2 CopyRow_C SME AVX AVX512BW AVX512BW CumulativeSumToAverageRow_C NEON AVX2 AVX2 AVX2 DetileRow_16_C NEON AVX2 AVX2 AVX2 DetileRow_C NEON AVX2 AVX2 AVX2 DetileSplitUVRow_C NEON AVX2 AVX2 AVX2 DetileToYUY2_C NEON AVX2 AVX2 AVX2 DivideRow_16_C SVE2 AVX2 AVX512BW AVX512BW GaussCol_C * NEON AVX2 AVX2 AVX2 GaussCol_F32_C NEON AVX2 AVX2 AVX2 GaussRow_C * NEON AVX2 AVX2 AVX2 GaussRow_F32_C NEON AVX2 AVX2 AVX2 HalfFloatRow_C SVE2 AVX2 AVX512BW AVX512BW HalfMergeUVRow_C SVE2 AVX2 AVX512BW AVX512BW HalfRow_16To8_C SME AVX2 AVX512BW AVX512BW HalfRow_16_C - AVX2 AVX2 AVX2 HalfRow_C - AVX2 AVX2 AVX2 HalfWidthRow_16To8_C SME AVX2 AVX512BW AVX512BW HalfWidthRow_16To8_Odd_C - AVX2 AVX2 AVX2 HammingDistance_C DOTPROD AVX2 AVX512BW AVX512BW HashDjb2_C NEON AVX2 AVX512BW AVX512BW I210AlphaToARGBRow_C SME AVX2 AVX512BW AVX512BW I210ToAR30Row_C SME AVX2 AVX512BW AVX512BW I210ToARGBRow_C SME AVX2 AVX512BW AVX512BW I212ToAR30Row_C SME AVX2 AVX512BW AVX512BW I212ToARGBRow_C SME AVX2 AVX512BW AVX512BW I400ToARGBRow_C SME AVX2 AVX512BW AVX512BW I410AlphaToARGBRow_C SME AVX2 AVX512BW AVX512BW I410ToAR30Row_C SME AVX2 AVX512BW AVX512BW I410ToARGBRow_C SME AVX2 AVX512BW AVX512BW I422AlphaToARGBRow_C SME AVX2 AVX512BW AVX512BW I422ToAR30Row_C SME AVX2 AVX512BW AVX512BW I422ToARGB1555Row_C SME AVX2 AVX512BW AVX512BW I422ToARGB4444Row_C SME AVX2 AVX512BW AVX512BW I422ToARGBRow_C SME AVX2 AVX512BW AVX512BW I422ToRGB24Row_C SME AVX2 AVX512VBMI AVX512VBMI I422ToRGB565Row_C SME AVX2 AVX512BW AVX512BW I422ToRGBARow_C SME AVX2 AVX512BW AVX512BW I422ToUYVYRow_C NEON AVX2 AVX2 AVX2 I422ToYUY2Row_C NEON AVX2 AVX2 AVX2 I444AlphaToARGBRow_C SME AVX2 AVX512BW AVX512BW I444ToARGBRow_C SME AVX2 AVX512BW AVX512BW I444ToRGB24Row_C SME AVX2 AVX512VBMI AVX512VBMI InterpolateRow_16_C SME AVX2 AVX512BW AVX512BW InterpolateRow_C SME AVX2 AVX512BW AVX512BW J400ToARGBRow_C NEON AVX2 AVX512BW AVX512BW MergeAR64Row_C NEON AVX2 AVX2 AVX2 MergeARGB16To8Row_C NEON AVX2 AVX2 AVX2 MergeARGBRow_C NEON AVX2 AVX2 AVX2 MergeRGBRow_C NEON AVX2 AVX512VBMI AVX512VBMI MergeUVRow_16_C SME AVX2 AVX512BW AVX512BW MergeUVRow_C SME AVX2 AVX512BW AVX512BW MergeXR30Row_C NEON AVX2 AVX512BW AVX512BW MergeXR64Row_C NEON AVX2 AVX2 AVX2 MergeXRGB16To8Row_C NEON AVX2 AVX2 AVX2 MergeXRGBRow_C NEON AVX2 AVX2 AVX2 MirrorRow_16_C NEON AVX2 AVX512BW AVX512BW MirrorRow_C NEON AVX2 AVX512BW AVX512BW MirrorSplitUVRow_C NEON AVX2 AVX512BW AVX512BW MirrorUVRow_C NEON AVX2 AVX512BW AVX512BW MultiplyRow_16_C SME AVX2 AVX512BW AVX512BW NV12ToARGBRow_C SME AVX2 AVX512BW AVX512BW NV12ToRGB24Row_C SME AVX2 AVX512VBMI AVX512VBMI NV12ToRGB565Row_C NEON AVX2 AVX2 AVX2 NV21ToARGBRow_C SME AVX2 AVX512BW AVX512BW NV21ToRGB24Row_C SME AVX2 AVX512VBMI AVX512VBMI NV21ToYUV24Row_C NEON AVX2 AVX2 AVX2 P210ToAR30Row_C SME AVX2 AVX512BW AVX512BW P210ToARGBRow_C SME AVX2 AVX512BW AVX512BW P410ToAR30Row_C SME AVX2 AVX512BW AVX512BW P410ToARGBRow_C SME AVX2 AVX512BW AVX512BW RAWToARGBRow_C SVE2 AVX2 AVX512BW AVX512BW RAWToRGB24Row_C SVE2 AVX2 AVX512VBMI AVX512VBMI RAWToRGBARow_C SVE2 AVX2 AVX512VBMI AVX512VBMI RGB24MirrorRow_C NEON AVX2 AVX512VBMI AVX512VBMI RGB24ToARGBRow_C SVE2 AVX2 AVX512BW AVX512BW RGB565ToARGBRow_C NEON AVX2 AVX2 AVX2 RGB565ToUVMatrixRow_C NEON AVX2 AVX2 AVX2 RGB565ToUVRow_C * NEON AVX2 AVX2 AVX2 RGB565ToYMatrixRow_C NEON AVX2 AVX2 AVX2 RGB565ToYRow_C * NEON AVX2 AVX2 AVX2 RGBAToARGBRow_C NEON AVX2 AVX512BW AVX512BW RGBColorTableRow_C NEON AVX2 AVX2 AVX2 RGBToUV444MatrixRow_C NEON AVX2 AVX512BW AVX512BW RGBToUVMatrixRow_C NEON AVX2 AVX512BW AVX512BW RGBToYMatrixRow_C DOTPROD AVX2 AVX512BW AVX512BW ScaleMaxSamples_C * NEON AVX2 AVX2 AVX2 ScaleSamples_C * NEON AVX2 AVX2 AVX2 ScaleSumSamples_C * NEON AVX2 AVX2 AVX2 SetRow_C NEON AVX2 AVX2 AVX2 SobelRow_C NEON AVX2 AVX2 AVX2 SobelToPlaneRow_C NEON AVX2 AVX2 AVX2 SobelXRow_C NEON AVX2 AVX2 AVX2 SobelXYRow_C NEON AVX2 AVX2 AVX2 SobelYRow_C NEON AVX2 AVX2 AVX2 SplitARGBRow_C NEON AVX2 AVX2 AVX2 SplitRGBRow_C NEON AVX2 AVX2 AVX2 SplitUVRow_16_C NEON AVX2 AVX2 AVX2 SplitUVRow_C SVE2 AVX2 AVX512BW AVX512BW SplitXRGBRow_C NEON AVX2 AVX2 AVX2 Ssim4x4Row_C SME AVX2 AVX512BW AVX512BW SsimWindowRow_C SME AVX2 AVX512BW AVX512BW SumSquareError_C DOTPROD AVX2 AVX512VNNI AVX512VNNI SwapUVRow_C NEON AVX2 AVX512BW AVX512BW TransposeUVWx16_C - AVX2 AVX512BW AVX512BW TransposeUVWx8_C NEON AVX2 AVX2 AVX2 TransposeWx16_C NEON AVX2 AVX512BW AVX512BW TransposeWx4_32_C * SVE2 AVX2 AVX512BW AVX512BW TransposeWx8_16_C SVE2 AVX2 AVX512BW AVX512BW TransposeWx8_C - AVX2 AVX2 AVX2 TransposeWxH_32_C NEON AVX2 AVX2 AVX2 UYVYToARGBRow_C SME AVX2 AVX512BW AVX512BW UYVYToUV422Row_C NEON AVX2 AVX2 AVX2 UYVYToUVRow_C SVE2 AVX2 AVX512BW AVX512BW UYVYToYRow_C SVE2 AVX2 AVX512BW AVX512BW UnpackMT2T_C NEON AVX2 AVX2 AVX2 YUY2ToARGBRow_C SME AVX2 AVX512BW AVX512BW YUY2ToNVUVRow_C NEON AVX2 AVX2 AVX2 YUY2ToUV422Row_C NEON AVX2 AVX2 AVX2 YUY2ToUVRow_C SVE2 AVX2 AVX512BW AVX512BW YUY2ToYRow_C SVE2 AVX2 AVX512BW AVX512BW ------------------------------------------------------------------------ SIMD (187 row/rot/cmp) 181 97% 187 100% 187 100% 187 100% ------------------------------------------------------------------------ Scale SIMD ISA supported per *_C function: ------------------------------------------------------------------------ C Function arm64 x86_32 x64 row_win ------------------------------------------------------------------------ FixedDiv1_C - X86 X86 X86 FixedDiv_C - X86 X86 X86 ScaleARGBCols64_C - - - - ScaleARGBColsUp2_C - AVX2 AVX2 AVX2 ScaleARGBCols_C NEON AVX2 AVX2 AVX2 ScaleARGBFilterCols64_C - - - - ScaleARGBFilterCols_C NEON AVX2 AVX2 AVX2 ScaleARGBRowDown2Box_C SME AVX2 AVX2 AVX2 ScaleARGBRowDown2Linear_C SME AVX2 AVX2 AVX2 ScaleARGBRowDown2_C SME AVX2 AVX2 AVX2 ScaleARGBRowDownEvenBox_C NEON AVX2 AVX2 AVX2 ScaleARGBRowDownEven_C NEON AVX2 AVX2 AVX2 ScaleAddCols0_C - - - - ScaleAddCols1_16_C - - - - ScaleAddCols1_C - - AVX512BW AVX512BW ScaleAddCols2_16_C - - - - ScaleAddCols2_C - - AVX512BW AVX512BW ScaleAddRow_16_C - - - - ScaleAddRow_C NEON AVX2 AVX512BW AVX512BW ScaleColsUp2_16_C - - - - ScaleColsUp2_C - AVX2 AVX2 AVX2 ScaleCols_16_C - - - - ScaleCols_C - - - - ScaleFilterCols64_16_C - - - - ScaleFilterCols64_C - - - - ScaleFilterCols_16_C - - - - ScaleFilterCols_C NEON AVX2 AVX2 AVX2 ScaleRowDown2Box_16_C SME AVX2 AVX2 AVX2 ScaleRowDown2Box_C SME AVX2 AVX512BW AVX512BW ScaleRowDown2Box_Odd_C - - AVX512BW AVX512BW ScaleRowDown2Linear_16_C SME AVX2 AVX2 AVX2 ScaleRowDown2Linear_C SME AVX2 AVX512BW AVX512BW ScaleRowDown2_16_C SME AVX2 AVX2 AVX2 ScaleRowDown2_C SME AVX2 AVX512BW AVX512BW ScaleRowDown34_0_Box_16_C - - - - ScaleRowDown34_0_Box_C NEON AVX2 AVX2 AVX2 ScaleRowDown34_16_C - - - - ScaleRowDown34_1_Box_16_C - - - - ScaleRowDown34_1_Box_C NEON AVX2 AVX2 AVX2 ScaleRowDown34_C NEON AVX2 AVX2 AVX2 ScaleRowDown38_16_C - - - - ScaleRowDown38_2_Box_16_C - - - - ScaleRowDown38_2_Box_C NEON AVX2 AVX2 AVX2 ScaleRowDown38_3_Box_16_C - - - - ScaleRowDown38_3_Box_C NEON AVX2 AVX2 AVX2 ScaleRowDown38_C NEON AVX2 AVX2 AVX2 ScaleRowDown4Box_16_C - - - - ScaleRowDown4Box_C NEON AVX2 AVX2 AVX2 ScaleRowDown4_16_C - - - - ScaleRowDown4_C NEON AVX2 AVX2 AVX2 ScaleRowUp2_Bilinear_16_Any_C NEON AVX2 AVX2 AVX2 ScaleRowUp2_Bilinear_16_C NEON AVX2 AVX2 AVX2 ScaleRowUp2_Bilinear_Any_C NEON AVX2 AVX2 AVX2 ScaleRowUp2_Bilinear_C NEON AVX2 AVX2 AVX2 ScaleRowUp2_Linear_16_Any_C NEON AVX2 AVX2 AVX2 ScaleRowUp2_Linear_16_C NEON AVX2 AVX2 AVX2 ScaleRowUp2_Linear_Any_C NEON AVX2 AVX2 AVX2 ScaleRowUp2_Linear_C NEON AVX2 AVX2 AVX2 ScaleUVAddCols1_16_C - - - - ScaleUVAddCols1_C - - - - ScaleUVAddCols2_16_C - - - - ScaleUVAddCols2_C - - - - ScaleUVAddCols3_16_C - - - - ScaleUVAddCols3_C SME AVX2 AVX512VBMI AVX512VBMI ScaleUVCols64_16_C - - - - ScaleUVCols64_C - - - - ScaleUVColsUp2_16_C - - - - ScaleUVColsUp2_C - - - - ScaleUVCols_16_C - - - - ScaleUVCols_C - - - - ScaleUVFilterCols64_16_C - - - - ScaleUVFilterCols64_C - - - - ScaleUVFilterCols_16_C - - - - ScaleUVFilterCols_C - - - - ScaleUVRowDown2Box_16_C - - - - ScaleUVRowDown2Box_C SME AVX2 AVX2 AVX2 ScaleUVRowDown2Linear_16_C - - - - ScaleUVRowDown2Linear_C SME AVX2 AVX2 AVX2 ScaleUVRowDown2_16_C - - - - ScaleUVRowDown2_C SME AVX2 AVX2 AVX2 ScaleUVRowDownEvenBox_16_C - - - - ScaleUVRowDownEvenBox_C - - - - ScaleUVRowDownEven_16_C - - - - ScaleUVRowDownEven_C NEON AVX2 AVX2 AVX2 ScaleUVRowUp2_Bilinear_16_Any_C NEON AVX2 AVX2 AVX2 ScaleUVRowUp2_Bilinear_16_C NEON AVX2 AVX2 AVX2 ScaleUVRowUp2_Bilinear_Any_C NEON AVX2 AVX2 AVX2 ScaleUVRowUp2_Bilinear_C NEON AVX2 AVX2 AVX2 ScaleUVRowUp2_Linear_16_Any_C NEON AVX2 AVX2 AVX2 ScaleUVRowUp2_Linear_16_C NEON AVX2 AVX2 AVX2 ScaleUVRowUp2_Linear_Any_C NEON AVX2 AVX2 AVX2 ScaleUVRowUp2_Linear_C NEON AVX2 AVX2 AVX2 ------------------------------------------------------------------------ SIMD (92 scale *_C) 44 48% 48 52% 51 55% 51 55% ------------------------------------------------------------------------ Test: libyuv_unittest Bug: none Change-Id: I2b4a832f2a7fe1fe93487b453240cd15c1f5f940 Reviewed-on: https://chromium-review.googlesource.com/c/libyuv/libyuv/+/8676400 Commit-Queue: Frank Barchard Reviewed-by: richard winterton --- GEMINI.md | 31 +- README.chromium | 2 +- docs/environment_variables.md | 2 + include/libyuv/compare_row.h | 41 +- include/libyuv/convert.h | 6 +- include/libyuv/convert_argb.h | 22 +- include/libyuv/convert_from_argb.h | 8 +- include/libyuv/cpu_id.h | 1 + include/libyuv/cpu_support.h | 2 +- include/libyuv/rotate_row.h | 112 +- include/libyuv/row.h | 768 +++- include/libyuv/scale_row.h | 365 +- include/libyuv/scale_uv.h | 1 - include/libyuv/version.h | 2 +- include/libyuv/video_common.h | 18 +- source/compare.cc | 52 +- source/compare_common.cc | 30 + source/compare_gcc.cc | 179 +- source/compare_win.cc | 89 +- source/convert.cc | 101 +- source/convert_argb.cc | 306 +- source/convert_from.cc | 5 + source/convert_from_argb.cc | 10 + source/convert_to_argb.cc | 5 +- source/convert_to_i420.cc | 22 +- source/cpu_id.cc | 7 +- source/planar_functions.cc | 291 +- source/rotate.cc | 127 +- source/rotate_any.cc | 23 +- source/rotate_argb.cc | 35 +- source/rotate_common.cc | 48 - source/rotate_gcc.cc | 1030 +++-- source/rotate_win.cc | 512 ++- source/row_any.cc | 146 +- source/row_common.cc | 142 +- source/row_gcc.cc | 5902 ++++++++++++++++++++++------ source/row_win.cc | 4997 ++++++++++++++++++++++- source/scale.cc | 207 +- source/scale_any.cc | 125 +- source/scale_argb.cc | 76 +- source/scale_common.cc | 200 +- source/scale_gcc.cc | 1619 +++++--- source/scale_uv.cc | 847 +++- source/scale_win.cc | 2658 ++++++++----- unit_test/compare_test.cc | 55 + unit_test/cpu_test.cc | 2 + unit_test/planar_test.cc | 428 +- unit_test/rotate_test.cc | 108 +- unit_test/scale_plane_test.cc | 7 + unit_test/scale_uv_test.cc | 285 ++ unit_test/unit_test.cc | 6 + util/cpuid.c | 2 + 52 files changed, 17498 insertions(+), 4567 deletions(-) diff --git a/GEMINI.md b/GEMINI.md index a82327f5b..923077777 100644 --- a/GEMINI.md +++ b/GEMINI.md @@ -47,11 +47,30 @@ and compiler compatibility. ### Utility and Fallbacks -* **row_common.cc**: Portable C/C++ versions. This is the reference - implementation. +* **row_common.cc**: Portable C/C++ versions and multi-step `ARGB` wrapper + row functions. This is the reference implementation. * **row_any.cc**: Handles "remainder" pixels for widths not multiples of SIMD - register size. Used for x86, Neon, and MIPS. Not required for SVE2, SME, - RVV, or HVX due to hardware-level masking. + register size. Used for SSE/AVX2, Neon, and LoongArch (LSX/LASX). Not + required for AVX-512, SVE2, SME, RVV, or HVX, which handle tail masking + directly inside the Row function. + +## Deprecated and Lower-Priority Formats + +The following datatypes and functions are deprecated or lower priority and are +not always fully hand-optimized across every SIMD extension: + +* **16-bit RGB formats** (deprecated): `RGB565` (`FOURCC_RGBP`, + `FOURCC_L565`), `ARGB1555` (`FOURCC_RGBO`, `FOURCC_L555`, `FOURCC_5551`), + and `ARGB4444` (`FOURCC_R444`). +* **Legacy fixed-matrix RGB-to-YUV row functions** (deprecated in favor of + `*MatrixRow` variants): `RGB565ToYRow`, `RGB565ToUVRow`, `ARGB1555ToYRow`, + `ARGB1555ToUVRow`, `ARGB4444ToYRow`, `ARGB4444ToUVRow`. +* **Row-biplanar YUV** (deprecated): `M420` (`FOURCC_M420`). + +When adding SIMD coverage for deprecated or less relevant formats, prefer +two-step wrapper implementations in `source/row_common.cc` that convert to or +from an intermediate stack-allocated `ARGB` row buffer (or delegate to existing +`*MatrixRow` functions) rather than writing large bespoke assembly kernels. ## Coding Guidelines @@ -72,7 +91,9 @@ and compiler compatibility. ### x86 Architectures (32-bit and 64-bit) 1. **AVX512 Logic**: AVX512 row functions are strictly enabled for **64-bit x86 - only**. + only**. Do not create `_Any_` wrappers for AVX512 row functions; handle + arbitrary tail widths directly inside the Row function using `k1`-`k7` + opmask registers. 2. **MSVC Intrinsics (`row_win.cc`)**: When converting `row_gcc.cc` to `row_win.cc`, use `_mm512_permutex2var_epi8`, not `_mm512_permi2var_epi8` intrinsic. diff --git a/README.chromium b/README.chromium index 6dfc0d43d..0bac66a43 100644 --- a/README.chromium +++ b/README.chromium @@ -1,6 +1,6 @@ Name: libyuv URL: https://chromium.googlesource.com/libyuv/libyuv/ -Version: 1976 +Version: 1977 Revision: DEPS License: BSD-3-Clause License File: LICENSE diff --git a/docs/environment_variables.md b/docs/environment_variables.md index a8c7aef6b..a46744e45 100644 --- a/docs/environment_variables.md +++ b/docs/environment_variables.md @@ -33,6 +33,8 @@ By default the cpu is detected and the most advanced form of SIMD is used. But LIBYUV_DISABLE_AVXVNNI LIBYUV_DISABLE_AVXVNNIINT8 LIBYUV_DISABLE_AMXINT8 + LIBYUV_DISABLE_AVX512BMM + LIBYUV_DISABLE_AMXTF32 ## Arm CPUs diff --git a/include/libyuv/compare_row.h b/include/libyuv/compare_row.h index 301a103e9..03caa5c01 100644 --- a/include/libyuv/compare_row.h +++ b/include/libyuv/compare_row.h @@ -21,14 +21,14 @@ namespace libyuv { extern "C" { #endif -// The following are available for 32 bit x86, Visual C 64 bit, and for clang -// with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These -// are intrinsics in compare_win.cc. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(_M_IX86) || \ - ((defined(__x86_64__) || defined(_M_X64)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)))) +// The following are available for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These are +// intrinsics in compare_win.cc. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) #define HAS_HAMMINGDISTANCE_SSE42 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HASHDJB2_SSE41 @@ -39,19 +39,26 @@ extern "C" { #define HAS_HASHDJB2_AVX2 #define HAS_SSIM4X4ROW_AVX2 #define HAS_SSIMWINDOWROW_AVX2 +#define HAS_ARGBDETECTROW_AVX2 +#if defined(__x86_64__) || defined(_M_X64) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW +#define HAS_SUMSQUAREERROR_AVX512VNNI #define HAS_HASHDJB2_AVX512BW #define HAS_SSIM4X4ROW_AVX512BW #define HAS_SSIMWINDOWROW_AVX512BW #endif +#endif -// The following are available for 64 bit GCC and clang, as inline assembly in -// compare_gcc.cc: -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) && \ +// The following are available for 32/64 bit GCC and clang, as inline assembly +// in compare_gcc.cc: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ (!defined(_MSC_VER) || defined(__clang__)) +#if defined(__x86_64__) #define HAS_HAMMINGDISTANCE_SSE42 +#endif #define HAS_HAMMINGDISTANCE_SSSE3 #define HAS_SUMSQUAREERROR_SSE2 #define HAS_HASHDJB2_SSE41 @@ -63,10 +70,12 @@ extern "C" { #define HAS_HASHDJB2_AVX2 #define HAS_SSIM4X4ROW_AVX2 #define HAS_SSIMWINDOWROW_AVX2 +#define HAS_ARGBDETECTROW_AVX2 #endif -#if defined(CLANG_HAS_AVX512) +#if defined(__x86_64__) && defined(CLANG_HAS_AVX512) #define HAS_HAMMINGDISTANCE_AVX512BW #define HAS_SUMSQUAREERROR_AVX512BW +#define HAS_SUMSQUAREERROR_AVX512VNNI #define HAS_HASHDJB2_AVX512BW #define HAS_SSIM4X4ROW_AVX512BW #define HAS_SSIMWINDOWROW_AVX512BW @@ -177,6 +186,9 @@ uint32_t SumSquareError_AVX2(const uint8_t* src_a, uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, const uint8_t* src_b, int count); +uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a, + const uint8_t* src_b, + int count); uint32_t SumSquareError_NEON(const uint8_t* src_a, const uint8_t* src_b, int count); @@ -298,6 +310,9 @@ float SsimWindowRow_RVV(const uint32_t* row0, const uint32_t* row1, int num_windows); +uint32_t ARGBDetectRow_C(const uint8_t* argb, int width); +uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width); + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/include/libyuv/convert.h b/include/libyuv/convert.h index adf35df7f..fdaec5eaf 100644 --- a/include/libyuv/convert.h +++ b/include/libyuv/convert.h @@ -1152,7 +1152,7 @@ int RAWToJ444(const uint8_t* src_raw, int width, int height); -// RGB16 (RGBP fourcc) little endian to I420. +// RGB16 (RGBP fourcc) little endian to I420. Deprecated. LIBYUV_API int RGB565ToI420(const uint8_t* src_rgb565, int src_stride_rgb565, @@ -1165,7 +1165,7 @@ int RGB565ToI420(const uint8_t* src_rgb565, int width, int height); -// RGB15 (RGBO fourcc) little endian to I420. +// RGB15 (RGBO fourcc) little endian to I420. Deprecated. LIBYUV_API int ARGB1555ToI420(const uint8_t* src_argb1555, int src_stride_argb1555, @@ -1178,7 +1178,7 @@ int ARGB1555ToI420(const uint8_t* src_argb1555, int width, int height); -// RGB12 (R444 fourcc) little endian to I420. +// RGB12 (R444 fourcc) little endian to I420. Deprecated. LIBYUV_API int ARGB4444ToI420(const uint8_t* src_argb4444, int src_stride_argb4444, diff --git a/include/libyuv/convert_argb.h b/include/libyuv/convert_argb.h index bcb2099d6..59b690f3a 100644 --- a/include/libyuv/convert_argb.h +++ b/include/libyuv/convert_argb.h @@ -1038,7 +1038,7 @@ int RAWToRGBA(const uint8_t* src_raw, int width, int height); -// RGB16 (RGBP fourcc) little endian to ARGB. +// RGB16 (RGBP fourcc) little endian to ARGB. Deprecated. LIBYUV_API int RGB565ToARGB(const uint8_t* src_rgb565, int src_stride_rgb565, @@ -1047,7 +1047,7 @@ int RGB565ToARGB(const uint8_t* src_rgb565, int width, int height); -// RGB15 (RGBO fourcc) little endian to ARGB. +// RGB15 (RGBO fourcc) little endian to ARGB. Deprecated. LIBYUV_API int ARGB1555ToARGB(const uint8_t* src_argb1555, int src_stride_argb1555, @@ -1056,7 +1056,7 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555, int width, int height); -// RGB12 (R444 fourcc) little endian to ARGB. +// RGB12 (R444 fourcc) little endian to ARGB. Deprecated. LIBYUV_API int ARGB4444ToARGB(const uint8_t* src_argb4444, int src_stride_argb4444, @@ -1173,7 +1173,7 @@ int Android420ToABGR(const uint8_t* src_y, int width, int height); -// Convert NV12 to RGB565. +// Convert NV12 to RGB565. Deprecated. LIBYUV_API int NV12ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1369,6 +1369,7 @@ int I422ToRAW(const uint8_t* src_y, int width, int height); +// Convert I420 to RGB565. Deprecated. LIBYUV_API int I420ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1381,6 +1382,7 @@ int I420ToRGB565(const uint8_t* src_y, int width, int height); +// Convert J420 to RGB565. Deprecated. LIBYUV_API int J420ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1393,6 +1395,7 @@ int J420ToRGB565(const uint8_t* src_y, int width, int height); +// Convert H420 to RGB565. Deprecated. LIBYUV_API int H420ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1405,6 +1408,7 @@ int H420ToRGB565(const uint8_t* src_y, int width, int height); +// Convert I422 to RGB565. Deprecated. LIBYUV_API int I422ToRGB565(const uint8_t* src_y, int src_stride_y, @@ -1417,7 +1421,7 @@ int I422ToRGB565(const uint8_t* src_y, int width, int height); -// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes). +// Convert I420 To RGB565 with 4x4 dither matrix (16 bytes). Deprecated. // Values in dither matrix from 0 to 7 recommended. // The order of the dither matrix is first byte is upper left. @@ -1434,6 +1438,7 @@ int I420ToRGB565Dither(const uint8_t* src_y, int width, int height); +// Convert I420 to ARGB1555. Deprecated. LIBYUV_API int I420ToARGB1555(const uint8_t* src_y, int src_stride_y, @@ -1446,6 +1451,7 @@ int I420ToARGB1555(const uint8_t* src_y, int width, int height); +// Convert I420 to ARGB4444. Deprecated. LIBYUV_API int I420ToARGB4444(const uint8_t* src_y, int src_stride_y, @@ -1903,7 +1909,7 @@ int NV21ToARGBMatrix(const uint8_t* src_y, int width, int height); -// Convert NV12 to RGB565 with matrix. +// Convert NV12 to RGB565 with matrix. Deprecated. LIBYUV_API int NV12ToRGB565Matrix(const uint8_t* src_y, int src_stride_y, @@ -2030,7 +2036,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y, int width, int height); -// Convert I420 to RGB565 with specified color matrix. +// Convert I420 to RGB565 with specified color matrix. Deprecated. LIBYUV_API int I420ToRGB565Matrix(const uint8_t* src_y, int src_stride_y, @@ -2044,7 +2050,7 @@ int I420ToRGB565Matrix(const uint8_t* src_y, int width, int height); -// Convert I422 to RGB565 with specified color matrix. +// Convert I422 to RGB565 with specified color matrix. Deprecated. LIBYUV_API int I422ToRGB565Matrix(const uint8_t* src_y, int src_stride_y, diff --git a/include/libyuv/convert_from_argb.h b/include/libyuv/convert_from_argb.h index 29d656d0a..817df6f9e 100644 --- a/include/libyuv/convert_from_argb.h +++ b/include/libyuv/convert_from_argb.h @@ -142,7 +142,7 @@ int ARGBToRAW(const uint8_t* src_argb, int width, int height); -// Convert ARGB To RGB565. +// Convert ARGB To RGB565. Deprecated. LIBYUV_API int ARGBToRGB565(const uint8_t* src_argb, int src_stride_argb, @@ -151,7 +151,7 @@ int ARGBToRGB565(const uint8_t* src_argb, int width, int height); -// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes). +// Convert ARGB To RGB565 with 4x4 dither matrix (16 bytes). Deprecated. // Values in dither matrix from 0 to 7 recommended. // The order of the dither matrix is first byte is upper left. // TODO(fbarchard): Consider pointer to 2d array for dither4x4. @@ -165,7 +165,7 @@ int ARGBToRGB565Dither(const uint8_t* src_argb, int width, int height); -// Convert ARGB To ARGB1555. +// Convert ARGB To ARGB1555. Deprecated. LIBYUV_API int ARGBToARGB1555(const uint8_t* src_argb, int src_stride_argb, @@ -174,7 +174,7 @@ int ARGBToARGB1555(const uint8_t* src_argb, int width, int height); -// Convert ARGB To ARGB4444. +// Convert ARGB To ARGB4444. Deprecated. LIBYUV_API int ARGBToARGB4444(const uint8_t* src_argb, int src_stride_argb, diff --git a/include/libyuv/cpu_id.h b/include/libyuv/cpu_id.h index 917df1cb4..54512b766 100644 --- a/include/libyuv/cpu_id.h +++ b/include/libyuv/cpu_id.h @@ -63,6 +63,7 @@ static const int kCpuHasAVXVNNI = 0x4000000; static const int kCpuHasAVXVNNIINT8 = 0x8000000; static const int kCpuHasAMXINT8 = 0x10000000; static const int kCpuHasAVX512BMM = 0x20000000; +static const int kCpuHasAMXTF32 = 0x40000000; // These flags are only valid on LOONGARCH processors. static const int kCpuHasLOONGARCH = 0x20; diff --git a/include/libyuv/cpu_support.h b/include/libyuv/cpu_support.h index adb50c42d..227bf3bd1 100644 --- a/include/libyuv/cpu_support.h +++ b/include/libyuv/cpu_support.h @@ -81,7 +81,7 @@ extern "C" { #endif // __clang__ // Visual C 2012 required for AVX2. -#if defined(_M_IX86) && \ +#if (defined(_M_IX86) || defined(_M_X64)) && \ (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) && \ defined(_MSC_VER) && _MSC_VER >= 1700 #define VISUALC_HAS_AVX2 1 diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index 4897256db..4ac43d388 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -21,19 +21,21 @@ namespace libyuv { extern "C" { #endif -// The following are available for 32 bit x86, Visual C 64 bit, and for clang -// with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These -// are intrinsics in rotate_win.cc. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(_M_IX86) || \ - ((defined(__x86_64__) || defined(_M_X64)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)))) -#define HAS_TRANSPOSEWX8_SSSE3 -#define HAS_TRANSPOSEUVWX8_SSE2 +// The following are available for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. These are +// intrinsics in rotate_win.cc. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) +#define HAS_TRANSPOSEWX8_AVX2 +#define HAS_TRANSPOSEUVWX8_AVX2 +#define HAS_TRANSPOSEWX8_16_AVX2 #define HAS_TRANSPOSEWX16_16_AVX2 -#define HAS_TRANSPOSEWX4_32_SSE2 +#define HAS_TRANSPOSEWX4_32_AVX2 #define HAS_TRANSPOSEWX8_32_AVX2 +#define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 #if defined(__x86_64__) || defined(_M_X64) @@ -44,22 +46,27 @@ extern "C" { #endif #endif -// The following are available for 64 bit GCC and clang, as inline assembly in -// rotate_gcc.cc: -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -#define HAS_TRANSPOSEWX8_SSSE3 -#define HAS_TRANSPOSEUVWX8_SSE2 +// The following are available for 32/64 bit GCC and clang, as inline assembly +// in rotate_gcc.cc: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ + (!defined(_MSC_VER) || defined(__clang__)) +#define HAS_TRANSPOSEWX8_AVX2 +#define HAS_TRANSPOSEUVWX8_AVX2 +#define HAS_TRANSPOSEWX8_16_AVX2 #define HAS_TRANSPOSEWX16_16_AVX2 -#define HAS_TRANSPOSEWX4_32_SSE2 +#define HAS_TRANSPOSEWX4_32_AVX2 #define HAS_TRANSPOSEWX8_32_AVX2 +#define HAS_TRANSPOSEWXH_32_AVX2 #define HAS_TRANSPOSEWX16_AVX2 #define HAS_TRANSPOSEUVWX16_AVX2 #endif // The following are available for AVX512 clang x64 platforms: #if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) + defined(CLANG_HAS_AVX512) && !defined(LIBYUV_ENABLE_ROWWIN) && \ + (!defined(_MSC_VER) || defined(__clang__)) #define HAS_TRANSPOSEWX16_AVX512BW #define HAS_TRANSPOSEUVWX16_AVX512BW #define HAS_TRANSPOSEWX32_16_AVX512BW @@ -128,13 +135,6 @@ extern "C" { #define HAS_TRANSPOSEWXH_32_HVX #endif -void TransposeWxH_C(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width, - int height); - void TransposeWx8_C(const uint8_t* src, int src_stride, uint8_t* dst, @@ -167,11 +167,11 @@ void TransposeWxH_HVX(const uint8_t* src, int dst_stride, int width, int height); -void TransposeWx8_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); +void TransposeWx8_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, @@ -213,11 +213,11 @@ void TransposeWx16_Any_NEON(const uint8_t* src, uint8_t* dst, int dst_stride, int width); -void TransposeWx8_Any_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); +void TransposeWx8_Any_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx16_Any_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, @@ -244,15 +244,6 @@ void TransposeWx16_Any_WASMSIMD(const uint8_t* src, int dst_stride, int width); -void TransposeUVWxH_C(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width, - int height); - void TransposeUVWx8_C(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -267,7 +258,7 @@ void TransposeUVWx16_C(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); -void TransposeUVWx8_SSE2(const uint8_t* src, +void TransposeUVWx8_AVX2(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, @@ -340,7 +331,7 @@ void TransposeUVWx16_WASMSIMD(const uint8_t* src, int dst_stride_b, int width); -void TransposeUVWx8_Any_SSE2(const uint8_t* src, +void TransposeUVWx8_Any_AVX2(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, @@ -389,18 +380,17 @@ void TransposeUVWx16_Any_WASMSIMD(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); -void TransposeWxH_16_C(const uint16_t* src, - int src_stride, - uint16_t* dst, - int dst_stride, - int width, - int height); void TransposeWx8_16_C(const uint16_t* src, int src_stride, uint16_t* dst, int dst_stride, int width); +void TransposeWx8_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); void TransposeWx16_16_AVX2(const uint16_t* src, int src_stride, uint16_t* dst, @@ -454,6 +444,11 @@ void TransposeWx8_16_WASMSIMD(const uint16_t* src, uint16_t* dst, int dst_stride, int width); +void TransposeWx8_16_Any_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width); void TransposeWx16_16_Any_AVX2(const uint16_t* src, int src_stride, uint16_t* dst, @@ -513,7 +508,7 @@ void TransposeWx4_32_SVE2(const uint8_t* src, int dst_stride, int width); -void TransposeWx4_32_SSE2(const uint8_t* src, +void TransposeWx4_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, @@ -525,6 +520,13 @@ void TransposeWx8_32_AVX2(const uint8_t* src, int dst_stride, int width); +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height); + void TransposeWx16_32_AVX512BW(const uint8_t* src, int src_stride, uint8_t* dst, @@ -574,7 +576,7 @@ void TransposeWx4_32_C(const uint8_t* src, int dst_stride, int width); -void TransposeWx4_32_Any_SSE2(const uint8_t* src, +void TransposeWx4_32_Any_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index f3c46bb2a..3fe8341b9 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -39,7 +39,6 @@ extern "C" { #if defined(USE_ROW_GCC) // Conversions: #define HAS_ARGBEXTRACTALPHAROW_SSE2 -#define HAS_ARGBSETROW_X86 #define HAS_ARGBSHUFFLEROW_SSSE3 #define HAS_ARGBTORAWROW_SSSE3 #define HAS_ARGBTORGB24ROW_SSSE3 @@ -66,7 +65,6 @@ extern "C" { #define HAS_RAWTORGB24ROW_SSSE3 #define HAS_RGB24TOARGBROW_SSSE3 #define HAS_SETROW_ERMS -#define HAS_SETROW_X86 #define HAS_SPLITUVROW_SSE2 #define HAS_UYVYTOARGBROW_SSSE3 #define HAS_UYVYTOYROW_SSE2 @@ -77,53 +75,17 @@ extern "C" { // Effects: #define HAS_ARGBADDROW_SSE2 -#define HAS_ARGBAFFINEROW_SSE2 #define HAS_ARGBALPHABLENDROW_SSSE3 -#define HAS_ARGBBLENDROW_SSSE3 -#define HAS_ARGBCOLORMATRIXROW_SSSE3 -#define HAS_ARGBCOLORTABLEROW_X86 #define HAS_ARGBCOPYALPHAROW_SSE2 #define HAS_ARGBCOPYYTOALPHAROW_SSE2 -#define HAS_ARGBGRAYROW_SSSE3 -#define HAS_ARGBLUMACOLORTABLEROW_SSSE3 #define HAS_ARGBMIRRORROW_SSE2 #define HAS_ARGBMULTIPLYROW_SSE2 #define HAS_ARGBORROW_SSE2 #define HAS_ARGBPOLYNOMIALROW_SSE2 -#define HAS_ARGBQUANTIZEROW_SSE2 -#define HAS_ARGBSEPIAROW_SSSE3 -#define HAS_ARGBSHADEROW_SSE2 #define HAS_ARGBSUBTRACTROW_SSE2 #define HAS_BLENDPLANEROW_SSSE3 -#define HAS_COMPUTECUMULATIVESUMROW_SSE2 -#define HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 -#define HAS_RGBCOLORTABLEROW_X86 -#define HAS_SOBELROW_SSE2 -#define HAS_SOBELTOPLANEROW_SSE2 -#define HAS_SOBELXROW_SSE2 -#define HAS_SOBELXYROW_SSE2 -#define HAS_SOBELYROW_SSE2 - -// The following functions fail on gcc/clang 32 bit with fpic and framepointer. -// caveat: clangcl uses row_win.cc which works. -#if (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \ - defined(_MSC_VER)) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -// TODO(fbarchard): fix build error on android_full_debug=1 -// https://code.google.com/p/libyuv/issues/detail?id=517 #define HAS_I422ALPHATOARGBROW_SSSE3 #define HAS_I444ALPHATOARGBROW_SSSE3 -#endif -#if (defined(__x86_64__) || !defined(__pic__) || defined(__clang__) || \ - defined(_MSC_VER)) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -// TODO(fbarchard): fix build error on android_full_debug=1 -// https://code.google.com/p/libyuv/issues/detail?id=517 -// TODO(fbarchard): fix LIBYUV_ENABLE_ROWWIN with clang -#define HAS_I422ALPHATOARGBROW_AVX2 -#define HAS_I444ALPHATOARGBROW_AVX2 -#endif - #endif // The following are available on all x86 platforms, but @@ -160,11 +122,13 @@ extern "C" { #define HAS_ARGBTORGB565DITHERROW_AVX2 #define HAS_COPYROW_AVX #define HAS_HALFFLOATROW_AVX2 +#define HAS_I422ALPHATOARGBROW_AVX2 #define HAS_I422TORGB24ROW_AVX2 #define HAS_I422TORGBAROW_AVX2 #define HAS_I422TORGB565ROW_AVX2 #define HAS_I422TOARGB1555ROW_AVX2 #define HAS_I422TOARGB4444ROW_AVX2 +#define HAS_I444ALPHATOARGBROW_AVX2 #define HAS_I444TOARGBROW_AVX2 #define HAS_I444TORGB24ROW_AVX2 #define HAS_J400TOARGBROW_AVX2 @@ -204,10 +168,6 @@ extern "C" { #define HAS_ARGBUNATTENUATEROW_SSE2 #define HAS_CONVERT16TO8ROW_SSSE3 #define HAS_CONVERT8TO16ROW_SSE2 -#define HAS_DETILEROW_16_SSE2 -#define HAS_DETILEROW_SSE2 -#define HAS_DETILESPLITUVROW_SSSE3 -#define HAS_DETILETOYUY2_SSE2 #define HAS_HALFMERGEUVROW_SSSE3 #define HAS_HALFROW_16TO8_SSSE3 #define HAS_HALFWIDTHROW_16TO8_SSSE3 @@ -218,11 +178,8 @@ extern "C" { #define HAS_I400TOARGBROW_SSE2 #define HAS_I410TOAR30ROW_SSSE3 #define HAS_I410TOARGBROW_SSSE3 -#if defined(__x86_64__) #define HAS_I422TOAR30ROW_SSSE3 -#endif #define HAS_MERGEARGBROW_SSE2 -#define HAS_MERGERGBROW_SSSE3 #define HAS_MERGEXRGBROW_SSE2 #define HAS_MIRRORUVROW_SSSE3 #define HAS_NV21TOYUV24ROW_SSSE3 @@ -236,7 +193,6 @@ extern "C" { #define HAS_RGB565TOARGBROW_AVX2 #define HAS_ARGB1555TOARGBROW_AVX2 #define HAS_ARGB4444TOARGBROW_AVX2 -#define HAS_RAWTORGBAROW_SSSE3 #define HAS_SPLITARGBROW_SSE2 #define HAS_SPLITARGBROW_SSSE3 #define HAS_SPLITRGBROW_SSE41 @@ -252,14 +208,9 @@ extern "C" { #define HAS_RGBTOUVMATRIXROW_SSSE3 #define HAS_ARGBTOUV444MATRIXROW_SSSE3 #define HAS_RGBTOUV444MATRIXROW_SSSE3 - -#if defined(__x86_64__) || !defined(__pic__) -// TODO(fbarchard): fix build error on android_full_debug=1 -// https://code.google.com/p/libyuv/issues/detail?id=517 #define HAS_I210ALPHATOARGBROW_SSSE3 #define HAS_I410ALPHATOARGBROW_SSSE3 #endif -#endif // The following are available for AVX2 gcc/clang x86 platforms: // TODO(fbarchard): Port to Visual C @@ -269,6 +220,9 @@ extern "C" { !defined(LIBYUV_ENABLE_ROWWIN) #define HAS_AB64TOARGBROW_AVX2 #define HAS_ABGRTOAR30ROW_AVX2 +#define HAS_AR30TOAB30ROW_AVX2 +#define HAS_AR30TOABGRROW_AVX2 +#define HAS_AR30TOARGBROW_AVX2 #define HAS_AR64TOARGBROW_AVX2 #define HAS_ARGBATTENUATEROW_AVX2 #define HAS_ARGBTOAB64ROW_AVX2 @@ -291,15 +245,58 @@ extern "C" { #define HAS_RGB565TOYMATRIXROW_AVX2 #define HAS_ARGB1555TOYMATRIXROW_AVX2 #define HAS_ARGB4444TOYMATRIXROW_AVX2 +#define HAS_ARGB1555TOUVROW_AVX2 +#define HAS_ARGB1555TOYROW_AVX2 +#define HAS_ARGB4444TOUVROW_AVX2 +#define HAS_ARGB4444TOYROW_AVX2 +#define HAS_ARGBAFFINEROW_AVX2 +#define HAS_ARGBCOLORMATRIXROW_AVX2 +#define HAS_ARGBCOLORTABLEROW_AVX2 +#define HAS_ARGBLUMACOLORTABLEROW_AVX2 +#define HAS_ARGBQUANTIZEROW_AVX2 +#define HAS_ARGBSEPIAROW_AVX2 +#define HAS_ARGBSETROW_AVX2 +#define HAS_ARGBSHADEROW_AVX2 #define HAS_ARGBUNATTENUATEROW_AVX2 +#define HAS_ARGBBLENDROW_AVX2 +#define HAS_ARGBGRAYROW_AVX2 +#define HAS_AYUVTOUVROW_AVX2 +#define HAS_AYUVTOVUROW_AVX2 +#define HAS_AYUVTOYROW_AVX2 +#define HAS_BYTETOFLOATROW_AVX2 +#define HAS_COMPUTECUMULATIVESUMROW_AVX2 #define HAS_CONVERT16TO8ROW_AVX2 #define HAS_CONVERT8TO16ROW_AVX2 #define HAS_CONVERT8TO8ROW_AVX2 -#define HAS_DETILEROW_16_AVX +#define HAS_COPYROW_16_AVX2 +#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 +#define HAS_DETILEROW_16_AVX2 +#define HAS_DETILEROW_AVX2 +#define HAS_DETILESPLITUVROW_AVX2 +#define HAS_DETILETOYUY2_AVX2 #define HAS_DIVIDEROW_16_AVX2 +#define HAS_GAUSSCOL_AVX2 +#define HAS_GAUSSCOL_F32_AVX2 +#define HAS_GAUSSROW_AVX2 +#define HAS_GAUSSROW_F32_AVX2 +#define HAS_RGB565TOUVROW_AVX2 +#define HAS_RGB565TOYROW_AVX2 +#define HAS_RGBCOLORTABLEROW_AVX2 +#define HAS_SCALEMAXSAMPLES_AVX2 +#define HAS_SCALESAMPLES_AVX2 +#define HAS_SCALESUMSAMPLES_AVX2 +#define HAS_SETROW_AVX2 +#define HAS_SOBELROW_AVX2 +#define HAS_SOBELTOPLANEROW_AVX2 +#define HAS_SOBELXROW_AVX2 +#define HAS_SOBELXYROW_AVX2 +#define HAS_SOBELYROW_AVX2 #define HAS_HALFMERGEUVROW_AVX2 +#define HAS_HALFROW_16_AVX2 #define HAS_HALFROW_16TO8_AVX2 +#define HAS_HALFROW_AVX2 #define HAS_HALFWIDTHROW_16TO8_AVX2 +#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2 #define HAS_I210TOAR30ROW_AVX2 #define HAS_I210TOARGBROW_AVX2 #define HAS_I212TOAR30ROW_AVX2 @@ -307,17 +304,15 @@ extern "C" { #define HAS_I400TOARGBROW_AVX2 #define HAS_I410TOAR30ROW_AVX2 #define HAS_I410TOARGBROW_AVX2 -#if defined(__x86_64__) -#define HAS_COMPUTECUMULATIVESUMROW_AVX2 -#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 #define HAS_I422TOAR30ROW_AVX2 -#endif #define HAS_RAWTORGB24ROW_AVX2 +#define HAS_RAWTORGBAROW_AVX2 #define HAS_I422TOUYVYROW_AVX2 #define HAS_I422TOYUY2ROW_AVX2 #define HAS_MERGEAR64ROW_AVX2 #define HAS_MERGEARGB16TO8ROW_AVX2 #define HAS_MERGEARGBROW_AVX2 +#define HAS_MERGERGBROW_AVX2 #define HAS_MERGEUVROW_16_AVX2 #define HAS_MERGEXR30ROW_AVX2 #define HAS_MERGEXR64ROW_AVX2 @@ -335,15 +330,11 @@ extern "C" { #define HAS_SPLITUVROW_16_AVX2 #define HAS_SPLITXRGBROW_AVX2 #define HAS_SWAPUVROW_AVX2 +#define HAS_UNPACKMT2T_AVX2 #define HAS_YUY2TONVUVROW_AVX2 - -#if defined(__x86_64__) || !defined(__pic__) -// TODO(fbarchard): fix build error on android_full_debug=1 -// https://code.google.com/p/libyuv/issues/detail?id=517 #define HAS_I210ALPHATOARGBROW_AVX2 #define HAS_I410ALPHATOARGBROW_AVX2 #endif -#endif // This module is for Visual C 32/64 bit #if !defined(LIBYUV_DISABLE_X86) && defined(USE_ROW_WIN) && \ @@ -369,8 +360,14 @@ extern "C" { #define HAS_I422TOARGB4444ROW_AVX2 #define HAS_NV12TORGB565ROW_AVX2 #define HAS_CONVERT8TO16ROW_AVX2 -#if defined(__x86_64__) || defined(_M_X64) #define HAS_I422TOAR30ROW_AVX2 +#define HAS_AR30TOAB30ROW_AVX2 +#define HAS_AR30TOABGRROW_AVX2 +#define HAS_AR30TOARGBROW_AVX2 +#define HAS_AYUVTOUVROW_AVX2 +#define HAS_AYUVTOVUROW_AVX2 +#define HAS_AYUVTOYROW_AVX2 +#if defined(__x86_64__) || defined(_M_X64) #define HAS_RAWTOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW #define HAS_RAWTORGB24ROW_AVX512BW @@ -421,7 +418,70 @@ extern "C" { #define HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI #define HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI #define HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI +#define HAS_I422TOARGBROW_AVX512BW +#define HAS_ARGB1555TOARGBROW_AVX512BW +#define HAS_ARGBCOLORMATRIXROW_AVX512VNNI +#define HAS_ARGBSEPIAROW_AVX512VNNI +#define HAS_ARGBTORGB565DITHERROW_AVX512BW +#define HAS_ARGBTORGB565ROW_AVX512BW +#define HAS_CONVERT8TO8ROW_AVX512BW +#define HAS_DIVIDEROW_16_AVX512BW +#define HAS_HALFFLOATROW_AVX512BW +#define HAS_I210ALPHATOARGBROW_AVX512BW +#define HAS_I210TOAR30ROW_AVX512BW +#define HAS_I210TOARGBROW_AVX512BW +#define HAS_I212TOAR30ROW_AVX512BW +#define HAS_I212TOARGBROW_AVX512BW +#define HAS_I410ALPHATOARGBROW_AVX512BW +#define HAS_I410TOAR30ROW_AVX512BW +#define HAS_I410TOARGBROW_AVX512BW +#define HAS_I422TOARGB1555ROW_AVX512BW +#define HAS_I422TOARGB4444ROW_AVX512BW +#define HAS_I422TORGB565ROW_AVX512BW +#define HAS_I422TORGBAROW_AVX512BW +#define HAS_I444ALPHATOARGBROW_AVX512BW +#define HAS_I444TOARGBROW_AVX512BW +#define HAS_I444TORGB24ROW_AVX512VBMI +#define HAS_MERGEUVROW_16_AVX512BW +#define HAS_P210TOAR30ROW_AVX512BW +#define HAS_P210TOARGBROW_AVX512BW +#define HAS_P410TOAR30ROW_AVX512BW +#define HAS_P410TOARGBROW_AVX512BW +#define HAS_UYVYTOARGBROW_AVX512BW +#define HAS_UYVYTOUVROW_AVX512BW +#define HAS_UYVYTOYROW_AVX512BW +#define HAS_YUY2TOARGBROW_AVX512BW +#define HAS_YUY2TOUVROW_AVX512BW +#define HAS_YUY2TOYROW_AVX512BW #endif +#define HAS_ARGB1555TOUVROW_AVX2 +#define HAS_ARGB1555TOYROW_AVX2 +#define HAS_ARGB4444TOUVROW_AVX2 +#define HAS_ARGB4444TOYROW_AVX2 +#define HAS_ARGBAFFINEROW_AVX2 +#define HAS_ARGBCOLORMATRIXROW_AVX2 +#define HAS_ARGBCOLORTABLEROW_AVX2 +#define HAS_ARGBLUMACOLORTABLEROW_AVX2 +#define HAS_ARGBQUANTIZEROW_AVX2 +#define HAS_ARGBSEPIAROW_AVX2 +#define HAS_ARGBSETROW_AVX2 +#define HAS_ARGBSHADEROW_AVX2 +#define HAS_BYTETOFLOATROW_AVX2 +#define HAS_COPYROW_16_AVX2 +#define HAS_DETILEROW_16_AVX2 +#define HAS_DETILEROW_AVX2 +#define HAS_DETILESPLITUVROW_AVX2 +#define HAS_GAUSSROW_AVX2 +#define HAS_GAUSSROW_F32_AVX2 +#define HAS_RGB565TOUVROW_AVX2 +#define HAS_RGB565TOYROW_AVX2 +#define HAS_RGBCOLORTABLEROW_AVX2 +#define HAS_SETROW_AVX2 +#define HAS_SOBELROW_AVX2 +#define HAS_SOBELTOPLANEROW_AVX2 +#define HAS_SOBELXROW_AVX2 +#define HAS_SOBELXYROW_AVX2 +#define HAS_SOBELYROW_AVX2 #define HAS_ARGBALPHABLENDROW_SSSE3 #define HAS_ARGBORROW_SSE2 #define HAS_ARGBALPHABLENDROW_AVX2 @@ -448,16 +508,91 @@ extern "C" { #define HAS_INTERPOLATEROW_16_AVX2 #define HAS_MULTIPLYROW_16_AVX2 #define HAS_SWAPUVROW_AVX2 +#define HAS_AB64TOARGBROW_AVX2 +#define HAS_AR64TOARGBROW_AVX2 +#define HAS_ARGBADDROW_AVX2 +#define HAS_ARGBBLENDROW_AVX2 +#define HAS_ARGBCOPYALPHAROW_AVX2 +#define HAS_ARGBGRAYROW_AVX2 +#define HAS_ARGBPOLYNOMIALROW_AVX2 +#define HAS_ARGBSUBTRACTROW_AVX2 +#define HAS_ARGBTOAB64ROW_AVX2 +#define HAS_ARGBTOAR64ROW_AVX2 +#define HAS_ARGBTORAWROW_AVX2 +#define HAS_ARGBTORGB24ROW_AVX2 +#define HAS_ARGBUNATTENUATEROW_AVX2 +#define HAS_COMPUTECUMULATIVESUMROW_AVX2 +#define HAS_CONVERT16TO8ROW_AVX2 +#define HAS_COPYROW_AVX +#define HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 +#define HAS_DETILETOYUY2_AVX2 +#define HAS_GAUSSCOL_AVX2 +#define HAS_GAUSSCOL_F32_AVX2 +#define HAS_HALFROW_16_AVX2 +#define HAS_HALFROW_16TO8_AVX2 +#define HAS_HALFROW_AVX2 +#define HAS_HALFWIDTHROW_16TO8_AVX2 +#define HAS_HALFWIDTHROW_16TO8_ODD_AVX2 +#define HAS_I422TOUYVYROW_AVX2 +#define HAS_I422TOYUY2ROW_AVX2 +#define HAS_I444TOARGBROW_AVX2 +#define HAS_I444TORGB24ROW_AVX2 +#define HAS_MERGEAR64ROW_AVX2 +#define HAS_MERGEARGB16TO8ROW_AVX2 +#define HAS_MERGEARGBROW_AVX2 +#define HAS_MERGERGBROW_AVX2 +#define HAS_MERGEUVROW_16_AVX2 +#define HAS_MERGEXR30ROW_AVX2 +#define HAS_MERGEXR64ROW_AVX2 +#define HAS_MERGEXRGB16TO8ROW_AVX2 +#define HAS_MERGEXRGBROW_AVX2 +#define HAS_NV12TORGB24ROW_AVX2 +#define HAS_NV21TORGB24ROW_AVX2 +#define HAS_NV21TOYUV24ROW_AVX2 +#define HAS_RAWTORGBAROW_AVX2 +#define HAS_SCALEMAXSAMPLES_AVX2 +#define HAS_SCALESAMPLES_AVX2 +#define HAS_SCALESUMSAMPLES_AVX2 +#define HAS_SPLITARGBROW_AVX2 +#define HAS_SPLITRGBROW_AVX2 +#define HAS_SPLITUVROW_16_AVX2 +#define HAS_SPLITUVROW_AVX2 +#define HAS_SPLITXRGBROW_AVX2 +#define HAS_UNPACKMT2T_AVX2 +#define HAS_UYVYTOUV422ROW_AVX2 +#define HAS_UYVYTOUVROW_AVX2 +#define HAS_UYVYTOYROW_AVX2 +#define HAS_YUY2TONVUVROW_AVX2 +#define HAS_YUY2TOUV422ROW_AVX2 +#define HAS_YUY2TOUVROW_AVX2 +#define HAS_YUY2TOYROW_AVX2 #if defined(__x86_64__) || defined(_M_X64) +#define HAS_ARGBTORAWROW_AVX512BW +#define HAS_ARGBTORGB24ROW_AVX512BW +#define HAS_ARGBTOUV444MATRIXROW_AVX512BW +#define HAS_ARGBTOUVMATRIXROW_AVX512BW +#define HAS_ARGBTOYMATRIXROW_AVX512BW +#define HAS_CONVERT16TO8ROW_AVX512BW +#define HAS_CONVERT8TO16ROW_AVX512BW +#define HAS_COPYROW_AVX512BW +#define HAS_HALFROW_16TO8_AVX512BW +#define HAS_HALFWIDTHROW_16TO8_AVX512BW +#define HAS_J400TOARGBROW_AVX512BW +#define HAS_MERGEUVROW_AVX512BW +#define HAS_MIRRORROW_AVX512BW #define HAS_MIRRORUVROW_AVX512BW +#define HAS_MIRRORSPLITUVROW_AVX512BW +#define HAS_MULTIPLYROW_16_AVX512BW +#define HAS_RGBTOUV444MATRIXROW_AVX512BW +#define HAS_RGBTOUVMATRIXROW_AVX512BW +#define HAS_SPLITUVROW_AVX512BW #endif #endif // The following are available for AVX512 clang x64 platforms: // TODO(fbarchard): Port to GCC and Visual C -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \ - !defined(LIBYUV_ENABLE_ROWWIN) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) #define HAS_COPYROW_AVX512BW #define HAS_RAWTOARGBROW_AVX512BW #define HAS_RGB24TOARGBROW_AVX512BW @@ -534,6 +669,40 @@ extern "C" { #define HAS_ARGBTOYMATRIXROW_AVX512VNNIVBMI #define HAS_ARGBTOUV444MATRIXROW_AVX512VNNIVBMI #define HAS_ARGBTOUVMATRIXROW_AVX512VNNIVBMI +#define HAS_ARGB1555TOARGBROW_AVX512BW +#define HAS_ARGBCOLORMATRIXROW_AVX512VNNI +#define HAS_ARGBSEPIAROW_AVX512VNNI +#define HAS_ARGBTORGB565DITHERROW_AVX512BW +#define HAS_ARGBTORGB565ROW_AVX512BW +#define HAS_CONVERT8TO8ROW_AVX512BW +#define HAS_DIVIDEROW_16_AVX512BW +#define HAS_HALFFLOATROW_AVX512BW +#define HAS_I210ALPHATOARGBROW_AVX512BW +#define HAS_I210TOAR30ROW_AVX512BW +#define HAS_I210TOARGBROW_AVX512BW +#define HAS_I212TOAR30ROW_AVX512BW +#define HAS_I212TOARGBROW_AVX512BW +#define HAS_I410ALPHATOARGBROW_AVX512BW +#define HAS_I410TOAR30ROW_AVX512BW +#define HAS_I410TOARGBROW_AVX512BW +#define HAS_I422TOARGB1555ROW_AVX512BW +#define HAS_I422TOARGB4444ROW_AVX512BW +#define HAS_I422TORGB565ROW_AVX512BW +#define HAS_I422TORGBAROW_AVX512BW +#define HAS_I444ALPHATOARGBROW_AVX512BW +#define HAS_I444TOARGBROW_AVX512BW +#define HAS_I444TORGB24ROW_AVX512VBMI +#define HAS_MERGEUVROW_16_AVX512BW +#define HAS_P210TOAR30ROW_AVX512BW +#define HAS_P210TOARGBROW_AVX512BW +#define HAS_P410TOAR30ROW_AVX512BW +#define HAS_P410TOARGBROW_AVX512BW +#define HAS_UYVYTOARGBROW_AVX512BW +#define HAS_UYVYTOUVROW_AVX512BW +#define HAS_UYVYTOYROW_AVX512BW +#define HAS_YUY2TOARGBROW_AVX512BW +#define HAS_YUY2TOUVROW_AVX512BW +#define HAS_YUY2TOYROW_AVX512BW #endif // The following are available on Neon platforms: @@ -1179,6 +1348,7 @@ extern "C" { #define HAS_YUY2TOYROW_HVX #endif +#undef SIMD_ALIGNED #if defined(_MSC_VER) && !defined(__CLR_VER) && !defined(__clang__) #if defined(VISUALC_HAS_AVX2) #define SIMD_ALIGNED(var) __declspec(align(32)) var @@ -2687,9 +2857,24 @@ void RGBToUV444MatrixRow_Any_AVX512BW(const uint8_t* src_rgb, int width, const struct ArgbConstants* c); +// Deprecated. void RGB565ToYRow_C(const uint8_t* src_rgb565, uint8_t* dst_y, int width); void ARGB1555ToYRow_C(const uint8_t* src_argb1555, uint8_t* dst_y, int width); void ARGB4444ToYRow_C(const uint8_t* src_argb4444, uint8_t* dst_y, int width); +void RGB565ToYRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_y, int width); +void ARGB1555ToYRow_AVX2(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width); +void ARGB4444ToYRow_AVX2(const uint8_t* src_argb4444, + uint8_t* dst_y, + int width); +void RGB565ToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGB1555ToYRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void ARGB4444ToYRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void RGB565ToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void ARGB1555ToYRow_Any_NEON(const uint8_t* src_ptr, @@ -2775,6 +2960,7 @@ void RAWToUVRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, int width); +// Deprecated. void RGB565ToUVRow_C(const uint8_t* src_rgb565, int src_stride_rgb565, uint8_t* dst_u, @@ -2790,6 +2976,36 @@ void ARGB4444ToUVRow_C(const uint8_t* src_argb4444, uint8_t* dst_u, uint8_t* dst_v, int width); +void RGB565ToUVRow_AVX2(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void ARGB1555ToUVRow_AVX2(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void ARGB4444ToUVRow_AVX2(const uint8_t* src_argb4444, + int src_stride_argb4444, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void RGB565ToUVRow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void ARGB1555ToUVRow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void ARGB4444ToUVRow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_AVX2(const uint8_t* src, uint8_t* dst, int width); @@ -2798,9 +3014,6 @@ void MirrorRow_NEON(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width); void MirrorRow_C(const uint8_t* src, uint8_t* dst, int width); -void MirrorRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); void MirrorRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -2814,9 +3027,6 @@ void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_uv, int width); void MirrorUVRow_C(const uint8_t* src_uv, uint8_t* dst_uv, int width); -void MirrorUVRow_Any_AVX512BW(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); void MirrorUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void MirrorUVRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -2973,11 +3183,11 @@ void DetileRow_Any_NEON(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width); -void DetileRow_SSE2(const uint8_t* src, +void DetileRow_AVX2(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width); -void DetileRow_Any_SSE2(const uint8_t* src, +void DetileRow_Any_AVX2(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width); @@ -2993,37 +3203,29 @@ void DetileRow_16_Any_NEON(const uint16_t* src, ptrdiff_t src_tile_stride, uint16_t* dst, int width); -void DetileRow_16_SSE2(const uint16_t* src, +void DetileRow_16_AVX2(const uint16_t* src, ptrdiff_t src_tile_stride, uint16_t* dst, int width); -void DetileRow_16_Any_SSE2(const uint16_t* src, +void DetileRow_16_Any_AVX2(const uint16_t* src, ptrdiff_t src_tile_stride, uint16_t* dst, int width); -void DetileRow_16_AVX(const uint16_t* src, - ptrdiff_t src_tile_stride, - uint16_t* dst, - int width); -void DetileRow_16_Any_AVX(const uint16_t* src, - ptrdiff_t src_tile_stride, - uint16_t* dst, - int width); void DetileSplitUVRow_C(const uint8_t* src_uv, ptrdiff_t src_tile_stride, uint8_t* dst_u, uint8_t* dst_v, int width); -void DetileSplitUVRow_SSSE3(const uint8_t* src_uv, - ptrdiff_t src_tile_stride, - uint8_t* dst_u, - uint8_t* dst_v, - int width); -void DetileSplitUVRow_Any_SSSE3(const uint8_t* src_uv, - ptrdiff_t src_tile_stride, - uint8_t* dst_u, - uint8_t* dst_v, - int width); +void DetileSplitUVRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); +void DetileSplitUVRow_Any_AVX2(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void DetileSplitUVRow_NEON(const uint8_t* src_uv, ptrdiff_t src_tile_stride, uint8_t* dst_u, @@ -3040,13 +3242,13 @@ void DetileToYUY2_C(const uint8_t* src_y, ptrdiff_t src_uv_tile_stride, uint8_t* dst_yuy2, int width); -void DetileToYUY2_SSE2(const uint8_t* src_y, +void DetileToYUY2_AVX2(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, uint8_t* dst_yuy2, int width); -void DetileToYUY2_Any_SSE2(const uint8_t* src_y, +void DetileToYUY2_Any_AVX2(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, @@ -3065,6 +3267,7 @@ void DetileToYUY2_Any_NEON(const uint8_t* src_y, uint8_t* dst_yuy2, int width); void UnpackMT2T_C(const uint8_t* src, uint16_t* dst, size_t size); +void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size); void UnpackMT2T_NEON(const uint8_t* src, uint16_t* dst, size_t size); void MergeUVRow_C(const uint8_t* src_u, const uint8_t* src_v, @@ -3228,11 +3431,11 @@ void MergeRGBRow_C(const uint8_t* src_r, const uint8_t* src_b, uint8_t* dst_rgb, int width); -void MergeRGBRow_SSSE3(const uint8_t* src_r, - const uint8_t* src_g, - const uint8_t* src_b, - uint8_t* dst_rgb, - int width); +void MergeRGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width); void MergeRGBRow_NEON(const uint8_t* src_r, const uint8_t* src_g, const uint8_t* src_b, @@ -3248,11 +3451,11 @@ void MergeRGBRow_AVX512VBMI(const uint8_t* src_r, const uint8_t* src_b, uint8_t* dst_rgb, int width); -void MergeRGBRow_Any_SSSE3(const uint8_t* y_buf, - const uint8_t* u_buf, - const uint8_t* v_buf, - uint8_t* dst_ptr, - int width); +void MergeRGBRow_Any_AVX2(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + int width); void MergeRGBRow_Any_NEON(const uint8_t* src_r, const uint8_t* src_g, const uint8_t* src_b, @@ -3646,6 +3849,11 @@ void MergeUVRow_16_AVX2(const uint16_t* src_u, uint16_t* dst_uv, int depth, int width); +void MergeUVRow_16_AVX512BW(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width); void MergeUVRow_16_Any_AVX2(const uint16_t* src_u, const uint16_t* src_v, uint16_t* dst_uv, @@ -3738,6 +3946,10 @@ void DivideRow_16_AVX2(const uint16_t* src_y, uint16_t* dst_y, int scale, int width); +void DivideRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width); void DivideRow_16_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, int scale, @@ -3849,6 +4061,32 @@ void Convert16To8Row_RVV(const uint16_t* src_y, int scale, int width); +void HalfRow_C(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width); +void HalfRow_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int width); + +void HalfRow_16_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width); +void HalfRow_16_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int width); + void HalfRow_16To8_C(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, @@ -3920,6 +4158,11 @@ void HalfWidthRow_16To8_Odd_C(const uint16_t* src_uv, uint8_t* dst_uv, int scale, int width); +void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width); void HalfWidthRow_16To8_SSSE3(const uint16_t* src_uv, ptrdiff_t src_uv_stride, uint8_t* dst_uv, @@ -4006,6 +4249,11 @@ void Convert8To8Row_AVX2(const uint8_t* src_y, int scale, int bias, int width); +void Convert8To8Row_AVX512BW(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width); void Convert8To8Row_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int scale, @@ -4030,6 +4278,8 @@ void CopyRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_Any_LASX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void CopyRow_16_C(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int count); +void CopyRow_16_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); void CopyRow_16_NEON(const uint16_t* src, uint16_t* dst, int count); void CopyRow_16_Any_NEON(const uint16_t* src_ptr, uint16_t* dst_ptr, int width); @@ -4112,20 +4362,21 @@ void ARGBCopyYToAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr, int width); void SetRow_C(uint8_t* dst, uint8_t v8, int width); -void SetRow_X86(uint8_t* dst, uint8_t v8, int width); void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width); +void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width); void SetRow_NEON(uint8_t* dst, uint8_t v8, int width); void SetRow_LSX(uint8_t* dst, uint8_t v8, int width); void SetRow_LASX(uint8_t* dst, uint8_t v8, int width); -void SetRow_Any_X86(uint8_t* dst_ptr, uint8_t v32, int width); +void SetRow_Any_AVX2(uint8_t* dst_ptr, uint8_t v32, int width); void SetRow_Any_NEON(uint8_t* dst_ptr, uint8_t v32, int width); void SetRow_Any_LSX(uint8_t* dst_ptr, uint8_t v32, int width); void SetRow_Any_LASX(uint8_t* dst_ptr, uint8_t v32, int width); void ARGBSetRow_C(uint8_t* dst_argb, uint32_t v32, int width); -void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width); +void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width); void ARGBSetRow_RVV(uint8_t* dst_argb, uint32_t v32, int width); +void ARGBSetRow_Any_AVX2(uint8_t* dst_ptr, uint32_t v32, int width); void ARGBSetRow_Any_NEON(uint8_t* dst_ptr, uint32_t v32, int width); void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width); @@ -4197,7 +4448,7 @@ void RGB24ToARGBRow_AVX2(const uint8_t* src_rgb24, void RAWToARGBRow_AVX512BW(const uint8_t* src_raw, uint8_t* dst_argb, int width); -void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width); +void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width); void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_AVX2(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); void RAWToRGB24Row_AVX512BW(const uint8_t* src_raw, @@ -4207,12 +4458,16 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, uint8_t* dst_rgb24, int width); +// Deprecated. void RGB565ToARGBRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_argb, int width); void ARGB1555ToARGBRow_AVX2(const uint8_t* src_argb1555, uint8_t* dst_argb, int width); +void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width); void ARGB4444ToARGBRow_AVX2(const uint8_t* src_argb4444, uint8_t* dst_argb, int width); @@ -4289,21 +4544,33 @@ void ARGB4444ToARGBRow_C(const uint8_t* src_argb4444, uint8_t* dst_argb, int width); void AR30ToARGBRow_C(const uint8_t* src_ar30, uint8_t* dst_argb, int width); +void AR30ToARGBRow_AVX2(const uint8_t* src_ar30, uint8_t* dst_argb, int width); void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_argb, int width); void AR30ToABGRRow_C(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); +void AR30ToABGRRow_AVX2(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); void ARGBToAR30Row_C(const uint8_t* src_argb, uint8_t* dst_ar30, int width); void AR30ToAB30Row_C(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToAB30Row_AVX2(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); void AR30ToARGBRow_NEON(const uint8_t* src_ar30, uint8_t* dst_argb, int width); void AR30ToABGRRow_NEON(const uint8_t* src_ar30, uint8_t* dst_abgr, int width); void AR30ToAB30Row_NEON(const uint8_t* src_ar30, uint8_t* dst_ab30, int width); +void AR30ToARGBRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToABGRRow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); +void AR30ToAB30Row_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void AR30ToARGBRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4327,9 +4594,9 @@ void RGB24ToARGBRow_Any_AVX2(const uint8_t* src_ptr, void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, uint8_t* dst_argb, int width); -void RAWToRGBARow_Any_SSSE3(const uint8_t* src_ptr, - uint8_t* dst_ptr, - int width); +void RAWToRGBARow_Any_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void RAWToRGB24Row_Any_SSSE3(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4417,6 +4684,7 @@ void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width); void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width); void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width); +// Deprecated. void ARGBToRGB565DitherRow_C(const uint8_t* src_argb, uint8_t* dst_rgb, uint32_t dither4, @@ -4426,8 +4694,15 @@ void ARGBToRGB565DitherRow_AVX2(const uint8_t* src, uint8_t* dst, uint32_t dither4, int width); +void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width); void ARGBToRGB565Row_AVX2(const uint8_t* src_argb, uint8_t* dst_rgb, int width); +void ARGBToRGB565Row_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width); void ARGBToARGB1555Row_AVX2(const uint8_t* src_argb, uint8_t* dst_rgb, int width); @@ -4806,24 +5081,49 @@ void I422ToRGBARow_AVX2(const uint8_t* y_buf, uint8_t* dst_rgba, const struct YuvConstants* yuvconstants, int width); +void I422ToRGBARow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width); +// Deprecated. void I422ToRGB565Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_rgb565, const struct YuvConstants* yuvconstants, int width); +void I422ToRGB565Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb565, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGB1555Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb1555, const struct YuvConstants* yuvconstants, int width); +void I422ToARGB1555Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb1555, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGB4444Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_argb4444, const struct YuvConstants* yuvconstants, int width); +void I422ToARGB4444Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb4444, + const struct YuvConstants* yuvconstants, + int width); void I422ToRGB565Row_SSSE3(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, @@ -4854,6 +5154,12 @@ void I444ToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I444ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I444ToRGB24Row_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -4866,6 +5172,12 @@ void I444ToRGB24Row_AVX2(const uint8_t* y_buf, uint8_t* dst_rgb24, const struct YuvConstants* yuvconstants, int width); +void I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -4941,36 +5253,72 @@ void I210ToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I210ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void I210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void I212ToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I212ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I212ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void I212ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void I410ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void I410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void I410ToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I210AlphaToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -4978,6 +5326,13 @@ void I210AlphaToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I410AlphaToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* u_buf, const uint16_t* v_buf, @@ -4985,6 +5340,13 @@ void I410AlphaToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -4999,6 +5361,13 @@ void I444AlphaToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -5112,10 +5481,18 @@ void YUY2ToARGBRow_AVX2(const uint8_t* yuy2_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void YUY2ToARGBRow_AVX512BW(const uint8_t* yuy2_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void UYVYToARGBRow_AVX2(const uint8_t* uyvy_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void UYVYToARGBRow_AVX512BW(const uint8_t* uyvy_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void P210ToARGBRow_SSSE3(const uint16_t* y_buf, const uint16_t* uv_buf, @@ -5142,21 +5519,41 @@ void P210ToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void P210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void P410ToARGBRow_AVX2(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void P410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void P210ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void P210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void P410ToAR30Row_AVX2(const uint16_t* y_buf, const uint16_t* uv_buf, uint8_t* dst_ar30, const struct YuvConstants* yuvconstants, int width); +void P410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width); void I422ToRGBARow_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, @@ -5604,10 +6001,10 @@ void I400ToARGBRow_Any_LASX(const uint8_t* src_ptr, int width); // ARGB preattenuated alpha blend. -void ARGBBlendRow_SSSE3(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width); +void ARGBBlendRow_AVX2(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); void ARGBBlendRow_AVX512BW(const uint8_t* src_argb, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -6498,6 +6895,7 @@ void YUY2ToUV422Row_SSE2(const uint8_t* src_yuy2, int width); void YUY2ToYRow_NEON(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToYRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width); +void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_u, @@ -6508,12 +6906,23 @@ void YUY2ToUVRow_SVE2(const uint8_t* src_yuy2, uint8_t* dst_u, uint8_t* dst_v, int width); +void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void UYVYToYRow_SVE2(const uint8_t* src_uyvy, uint8_t* dst_y, int width); +void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width); void UYVYToUVRow_SVE2(const uint8_t* src_uyvy, int stride_uyvy, uint8_t* dst_u, uint8_t* dst_v, int width); +void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void YUY2ToNVUVRow_NEON(const uint8_t* src_yuy2, int stride_yuy2, uint8_t* dst_uv, @@ -6749,11 +7158,16 @@ void SwapUVRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void SwapUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_vu, int width); void SwapUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_vu, int width); void AYUVToYRow_C(const uint8_t* src_ayuv, uint8_t* dst_y, int width); +void AYUVToYRow_AVX2(const uint8_t* src_ayuv, uint8_t* dst_y, int width); void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width); void AYUVToUVRow_C(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, int width); +void AYUVToUVRow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width); void AYUVToUVRow_AVX512VBMI(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_uv, @@ -6762,6 +7176,10 @@ void AYUVToVURow_C(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, int width); +void AYUVToVURow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width); void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, @@ -6783,6 +7201,15 @@ void AYUVToVURow_SVE2(const uint8_t* src_ayuv, int src_stride_ayuv, uint8_t* dst_vu, int width); +void AYUVToYRow_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void AYUVToUVRow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_vu, + int width); +void AYUVToVURow_Any_AVX2(const uint8_t* src_ptr, + int src_stride, + uint8_t* dst_vu, + int width); void AYUVToYRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void AYUVToUVRow_Any_NEON(const uint8_t* src_ptr, int src_stride, @@ -6976,7 +7403,7 @@ void ARGBUnattenuateRow_Any_NEON(const uint8_t* src_ptr, int width); void ARGBGrayRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width); -void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width); +void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, int width); @@ -6991,7 +7418,8 @@ void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width); void ARGBSepiaRow_C(uint8_t* dst_argb, int width); -void ARGBSepiaRow_SSSE3(uint8_t* dst_argb, int width); +void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width); +void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width); void ARGBSepiaRow_NEON(uint8_t* dst_argb, int width); void ARGBSepiaRow_NEON_DotProd(uint8_t* dst_argb, int width); void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width); @@ -7001,10 +7429,14 @@ void ARGBColorMatrixRow_C(const uint8_t* src_argb, uint8_t* dst_argb, const int8_t* matrix_argb, int width); -void ARGBColorMatrixRow_SSSE3(const uint8_t* src_argb, - uint8_t* dst_argb, - const int8_t* matrix_argb, - int width); +void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width); +void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width); void ARGBColorMatrixRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, const int8_t* matrix_argb, @@ -7025,9 +7457,9 @@ void ARGBColorMatrixRow_LASX(const uint8_t* src_argb, void ARGBColorTableRow_C(uint8_t* dst_argb, const uint8_t* table_argb, int width); -void ARGBColorTableRow_X86(uint8_t* dst_argb, - const uint8_t* table_argb, - int width); +void ARGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width); void ARGBColorTableRow_NEON(uint8_t* dst_argb, const uint8_t* table_argb, int width); @@ -7035,9 +7467,9 @@ void ARGBColorTableRow_NEON(uint8_t* dst_argb, void RGBColorTableRow_C(uint8_t* dst_argb, const uint8_t* table_argb, int width); -void RGBColorTableRow_X86(uint8_t* dst_argb, - const uint8_t* table_argb, - int width); +void RGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width); void RGBColorTableRow_NEON(uint8_t* dst_argb, const uint8_t* table_argb, int width); @@ -7047,7 +7479,7 @@ void ARGBQuantizeRow_C(uint8_t* dst_argb, int interval_size, int interval_offset, int width); -void ARGBQuantizeRow_SSE2(uint8_t* dst_argb, +void ARGBQuantizeRow_AVX2(uint8_t* dst_argb, int scale, int interval_size, int interval_offset, @@ -7072,7 +7504,7 @@ void ARGBShadeRow_C(const uint8_t* src_argb, uint8_t* dst_argb, int width, uint32_t value); -void ARGBShadeRow_SSE2(const uint8_t* src_argb, +void ARGBShadeRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width, uint32_t value); @@ -7090,16 +7522,6 @@ void ARGBShadeRow_LASX(const uint8_t* src_argb, uint32_t value); // Used for blur. -void CumulativeSumToAverageRow_SSE2(const int32_t* topleft, - const int32_t* botleft, - int width, - int area, - uint8_t* dst, - int count); -void ComputeCumulativeSumRow_SSE2(const uint8_t* row, - int32_t* cumsum, - const int32_t* previous_cumsum, - int width); void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, const int32_t* botleft, int width, @@ -7139,7 +7561,7 @@ void ARGBAffineRow_C(const uint8_t* src_argb, const float* uv_dudv, int width); LIBYUV_API -void ARGBAffineRow_SSE2(const uint8_t* src_argb, +void ARGBAffineRow_AVX2(const uint8_t* src_argb, int src_argb_stride, uint8_t* dst_argb, const float* src_dudv, @@ -7260,7 +7682,7 @@ void SobelXRow_C(const uint8_t* src_y0, const uint8_t* src_y2, uint8_t* dst_sobelx, int width); -void SobelXRow_SSE2(const uint8_t* src_y0, +void SobelXRow_AVX2(const uint8_t* src_y0, const uint8_t* src_y1, const uint8_t* src_y2, uint8_t* dst_sobelx, @@ -7274,7 +7696,7 @@ void SobelYRow_C(const uint8_t* src_y0, const uint8_t* src_y1, uint8_t* dst_sobely, int width); -void SobelYRow_SSE2(const uint8_t* src_y0, +void SobelYRow_AVX2(const uint8_t* src_y0, const uint8_t* src_y1, uint8_t* dst_sobely, int width); @@ -7286,7 +7708,7 @@ void SobelRow_C(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); -void SobelRow_SSE2(const uint8_t* src_sobelx, +void SobelRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); @@ -7306,7 +7728,7 @@ void SobelToPlaneRow_C(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, int width); -void SobelToPlaneRow_SSE2(const uint8_t* src_sobelx, +void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, int width); @@ -7326,7 +7748,7 @@ void SobelXYRow_C(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); -void SobelXYRow_SSE2(const uint8_t* src_sobelx, +void SobelXYRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); @@ -7342,7 +7764,7 @@ void SobelXYRow_LASX(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); -void SobelRow_Any_SSE2(const uint8_t* y_buf, +void SobelRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); @@ -7358,7 +7780,7 @@ void SobelRow_Any_LASX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); -void SobelToPlaneRow_Any_SSE2(const uint8_t* y_buf, +void SobelToPlaneRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); @@ -7374,7 +7796,7 @@ void SobelToPlaneRow_Any_LASX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); -void SobelXYRow_Any_SSE2(const uint8_t* y_buf, +void SobelXYRow_Any_AVX2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); @@ -7418,6 +7840,10 @@ void HalfFloatRow_AVX2(const uint16_t* src, uint16_t* dst, float scale, int width); +void HalfFloatRow_AVX512BW(const uint16_t* src, + uint16_t* dst, + float scale, + int width); void HalfFloatRow_Any_AVX2(const uint16_t* src_ptr, uint16_t* dst_ptr, float param, @@ -7471,6 +7897,14 @@ void HalfFloatRow_Any_LASX(const uint16_t* src_ptr, float param, int width); void ByteToFloatRow_C(const uint8_t* src, float* dst, float scale, int width); +void ByteToFloatRow_AVX2(const uint8_t* src, + float* dst, + float scale, + int width); +void ByteToFloatRow_Any_AVX2(const uint8_t* src_ptr, + float* dst_ptr, + float param, + int width); void ByteToFloatRow_NEON(const uint8_t* src, float* dst, float scale, @@ -7497,11 +7931,11 @@ void ARGBLumaColorTableRow_C(const uint8_t* src_argb, int width, const uint8_t* luma, uint32_t lumacoeff); -void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb, - uint8_t* dst_argb, - int width, - const uint8_t* luma, - uint32_t lumacoeff); +void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff); void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb, uint8_t* dst_argb, int width, @@ -7509,21 +7943,39 @@ void ARGBLumaColorTableRow_NEON(const uint8_t* src_argb, uint32_t lumacoeff); float ScaleMaxSamples_C(const float* src, float* dst, float scale, int width); +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width); float ScaleMaxSamples_NEON(const float* src, float* dst, float scale, int width); float ScaleSumSamples_C(const float* src, float* dst, float scale, int width); +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width); float ScaleSumSamples_NEON(const float* src, float* dst, float scale, int width); void ScaleSamples_C(const float* src, float* dst, float scale, int width); +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width); void ScaleSamples_NEON(const float* src, float* dst, float scale, int width); +void GaussRow_F32_AVX2(const float* src, float* dst, int width); void GaussRow_F32_NEON(const float* src, float* dst, int width); void GaussRow_F32_C(const float* src, float* dst, int width); +void GaussCol_F32_AVX2(const float* src0, + const float* src1, + const float* src2, + const float* src3, + const float* src4, + float* dst, + int width); + void GaussCol_F32_NEON(const float* src0, const float* src1, const float* src2, @@ -7541,6 +7993,7 @@ void GaussCol_F32_C(const float* src0, int width); void GaussRow_C(const uint32_t* src, uint16_t* dst, int width); +void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width); void GaussRow_NEON(const uint32_t* src, uint16_t* dst, int width); void GaussCol_C(const uint16_t* src0, const uint16_t* src1, @@ -7549,6 +8002,13 @@ void GaussCol_C(const uint16_t* src0, const uint16_t* src4, uint32_t* dst, int width); +void GaussCol_AVX2(const uint16_t* src0, + const uint16_t* src1, + const uint16_t* src2, + const uint16_t* src3, + const uint16_t* src4, + uint32_t* dst, + int width); void GaussCol_NEON(const uint16_t* src0, const uint16_t* src1, const uint16_t* src2, diff --git a/include/libyuv/scale_row.h b/include/libyuv/scale_row.h index e58962e7e..a4d34c515 100644 --- a/include/libyuv/scale_row.h +++ b/include/libyuv/scale_row.h @@ -23,31 +23,26 @@ extern "C" { #endif // The following are available on all x86 platforms: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(_M_IX86) || ((defined(__x86_64__) || defined(__i386__)) && \ - !defined(LIBYUV_ENABLE_ROWWIN))) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(_M_IX86) || defined(_M_X64) || defined(__x86_64__) || \ + defined(__i386__)) #define HAS_FIXEDDIV1_X86 #define HAS_FIXEDDIV_X86 -#define HAS_SCALEARGBCOLS_SSE2 -#define HAS_SCALEARGBCOLSUP2_SSE2 -#define HAS_SCALEARGBFILTERCOLS_SSSE3 -#define HAS_SCALEARGBROWDOWN2_SSE2 -#define HAS_SCALEARGBROWDOWNEVEN_SSE2 -#define HAS_SCALECOLSUP2_SSE2 -#define HAS_SCALEFILTERCOLS_SSSE3 +#define HAS_SCALEADDROW_SSE2 +#endif + +// The following are available on 32-bit Visual C and gcc/clang x86 platforms: +#if !defined(LIBYUV_DISABLE_X86) && !defined(LIBYUV_ENABLE_ROWWIN) && \ + ((defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__)) || \ + defined(__x86_64__) || defined(__i386__)) #define HAS_SCALEROWDOWN2_SSSE3 -#define HAS_SCALEROWDOWN34_SSSE3 #define HAS_SCALEROWDOWN4_SSSE3 #endif // The following are available for gcc/clang x86 platforms: -// TODO(fbarchard): Port to Visual C #if !defined(LIBYUV_DISABLE_X86) && \ (defined(__x86_64__) || defined(__i386__)) && \ !defined(LIBYUV_ENABLE_ROWWIN) -#define HAS_SCALEADDROW_SSE2 -#define HAS_SCALEROWDOWN38_SSSE3 -#define HAS_SCALEUVROWDOWN2BOX_SSSE3 #define HAS_SCALEROWUP2_LINEAR_SSE2 #define HAS_SCALEROWUP2_LINEAR_SSSE3 #define HAS_SCALEROWUP2_BILINEAR_SSE2 @@ -62,66 +57,57 @@ extern "C" { #define HAS_SCALEUVROWUP2_BILINEAR_16_SSE41 #endif -// The following are available for gcc/clang x86 platforms, but -// require clang 3.4 or gcc 4.7. -// TODO(fbarchard): Port to Visual C -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__)) && \ - (defined(CLANG_HAS_AVX2) || defined(GCC_HAS_AVX2)) && \ - !defined(LIBYUV_ENABLE_ROWWIN) -#define HAS_SCALEADDROW_AVX2 -#define HAS_SCALEUVADDCOLS3_AVX2 -#define HAS_SCALEUVROWDOWN2BOX_AVX2 -#define HAS_SCALEROWUP2_LINEAR_AVX2 -#define HAS_SCALEROWUP2_BILINEAR_AVX2 -#define HAS_SCALEROWUP2_LINEAR_12_AVX2 -#define HAS_SCALEROWUP2_BILINEAR_12_AVX2 -#define HAS_SCALEROWUP2_LINEAR_16_AVX2 -#define HAS_SCALEROWUP2_BILINEAR_16_AVX2 -#define HAS_SCALEUVROWUP2_LINEAR_AVX2 -#define HAS_SCALEUVROWUP2_BILINEAR_AVX2 -#define HAS_SCALEUVROWUP2_LINEAR_16_AVX2 -#define HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 -#endif - // The following are available on all x86 platforms, but // require VS2012, clang 3.4 or gcc 4.7. -// The code supports NaCL but requires a new compiler and validator. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \ - defined(GCC_HAS_AVX2)) +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(VISUALC_HAS_AVX2) || defined(CLANG_HAS_AVX2) || \ + defined(GCC_HAS_AVX2) || \ + (defined(LIBYUV_ENABLE_ROWWIN) && \ + (defined(__x86_64__) || defined(__i386__) || \ + defined(_M_X64) || defined(_M_IX86)))) +#define HAS_SCALEADDROW_AVX2 +#define HAS_SCALEARGBCOLS_AVX2 +#define HAS_SCALEARGBCOLSUP2_AVX2 +#define HAS_SCALEARGBFILTERCOLS_AVX2 +#define HAS_SCALEARGBROWDOWN2_AVX2 +#define HAS_SCALEARGBROWDOWNEVEN_AVX2 +#define HAS_SCALECOLSUP2_AVX2 +#define HAS_SCALEFILTERCOLS_AVX2 +#define HAS_SCALEROWDOWN2_16_AVX2 #define HAS_SCALEROWDOWN2_AVX2 +#define HAS_SCALEROWDOWN34_AVX2 +#define HAS_SCALEROWDOWN38_AVX2 #define HAS_SCALEROWDOWN4_AVX2 +#define HAS_SCALEROWUP2_BILINEAR_12_AVX2 +#define HAS_SCALEROWUP2_BILINEAR_16_AVX2 +#define HAS_SCALEROWUP2_BILINEAR_AVX2 +#define HAS_SCALEROWUP2_LINEAR_12_AVX2 +#define HAS_SCALEROWUP2_LINEAR_16_AVX2 +#define HAS_SCALEROWUP2_LINEAR_AVX2 +#define HAS_SCALEUVADDCOLS3_AVX2 +#define HAS_SCALEUVROWDOWN2_AVX2 +#define HAS_SCALEUVROWDOWN2BOX_AVX2 +#define HAS_SCALEUVROWDOWN2LINEAR_AVX2 +#define HAS_SCALEUVROWDOWNEVEN_AVX2 +#define HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 +#define HAS_SCALEUVROWUP2_BILINEAR_AVX2 +#define HAS_SCALEUVROWUP2_LINEAR_16_AVX2 +#define HAS_SCALEUVROWUP2_LINEAR_AVX2 #endif -// The following are available for AVX512 clang x64 platforms: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(_M_X64)) && defined(CLANG_HAS_AVX512) && \ - !defined(LIBYUV_ENABLE_ROWWIN) +// The following are available for AVX512 x64 platforms: +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(_M_X64)) && \ + (defined(CLANG_HAS_AVX512) || \ + (defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) #define HAS_SCALEADDROW_AVX512BW -#define HAS_SCALEADDCOLS_AVX512BW +#define HAS_SCALEADDCOLS1_AVX512BW +#define HAS_SCALEADDCOLS2_AVX512BW #define HAS_SCALEUVADDCOLS3_AVX512VBMI #define HAS_SCALEROWDOWN2_AVX512BW #endif -// The following are available for Visual C 32/64 bit, and for clang with -// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer: -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ - defined(_M_IX86)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)) -#define HAS_SCALEADDROW_SSE2 -#define HAS_SCALEROWDOWN38_SSSE3 -#define HAS_SCALEADDROW_AVX2 -#define HAS_SCALEUVADDCOLS3_AVX2 -#if defined(__x86_64__) || defined(_M_X64) -#define HAS_SCALEADDROW_AVX512BW -#define HAS_SCALEADDCOLS_AVX512BW -#define HAS_SCALEUVADDCOLS3_AVX512VBMI -#endif -#endif - // The following are available on Neon platforms: #if !defined(LIBYUV_DISABLE_NEON) && \ (defined(__ARM_NEON__) || defined(LIBYUV_NEON) || defined(__aarch64__)) @@ -532,24 +518,46 @@ void ScaleUVRowDown2_C(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDown2_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowDown2Linear_C(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDown2Linear_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowDown2Box_C(const uint8_t* src_uv, ptrdiff_t src_stride, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDown2Box_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowDownEven_C(const uint8_t* src_uv, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDownEven_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowDownEvenBox_C(const uint8_t* src_uv, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDownEvenBox_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint16_t* dst_uv, + int dst_width); void ScaleUVRowUp2_Linear_C(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -589,30 +597,59 @@ void ScaleUVCols_C(uint8_t* dst_uv, int dst_width, int x, int dx); +void ScaleUVCols_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx); void ScaleUVCols64_C(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int x32, int dx); +void ScaleUVCols64_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x32, + int dx); void ScaleUVColsUp2_C(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int, int); +void ScaleUVColsUp2_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int, + int); void ScaleUVFilterCols_C(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int x, int dx); +void ScaleUVFilterCols_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx); void ScaleUVFilterCols64_C(uint8_t* dst_uv, const uint8_t* src_uv, int dst_width, int x32, int dx); +void ScaleUVFilterCols64_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x32, + int dx); void ScaleUVAddCols3_C(int dst_width, int scaleval, const uint16_t* src_uv, uint8_t* dst_uv); +void ScaleUVAddCols3_16_C(int dst_width, + int scaleval, + const uint32_t* src_uv, + uint16_t* dst_uv); // Specialized scalers for x86. void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, @@ -639,6 +676,18 @@ void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); +void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); +void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -672,30 +721,30 @@ void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width); -void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); +void ScaleRowDown34_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); void ScaleRowUp2_Linear_SSE2(const uint8_t* src_ptr, uint8_t* dst_ptr, @@ -842,6 +891,18 @@ void ScaleRowDown2Box_Odd_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +void ScaleRowDown2_16_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); +void ScaleRowDown2Linear_16_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); +void ScaleRowDown2Box_16_Any_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width); void ScaleRowDown4_Any_SSSE3(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -859,30 +920,30 @@ void ScaleRowDown4Box_Any_AVX2(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width); -void ScaleRowDown34_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown34_1_Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown34_0_Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_3_Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); -void ScaleRowDown38_2_Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); +void ScaleRowDown34_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_1_Box_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown34_0_Box_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_3_Box_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleRowDown38_2_Box_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); void ScaleAddRow_SSE2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); void ScaleAddRow_AVX2(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width); @@ -908,29 +969,29 @@ void ScaleAddCols2_AVX512BW(int dst_width, const uint16_t* src_ptr, uint8_t* dst_ptr); -void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, - const uint8_t* src_ptr, - int dst_width, - int x, - int dx); -void ScaleColsUp2_SSE2(uint8_t* dst_ptr, +void ScaleFilterCols_AVX2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx); +void ScaleColsUp2_AVX2(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, int x, int dx); // ARGB Column functions -void ScaleARGBCols_SSE2(uint8_t* dst_argb, +void ScaleARGBCols_AVX2(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, int dx); -void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx); -void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, +void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx); +void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, @@ -962,15 +1023,15 @@ void ScaleARGBFilterCols_RVV(uint8_t* dst_argb, int dx); // ARGB Row functions -void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); @@ -1022,15 +1083,15 @@ void ScaleARGBRowDown2Box_LSX(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDown2_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDown2_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); -void ScaleARGBRowDown2Linear_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDown2Linear_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); -void ScaleARGBRowDown2Box_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDown2Box_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); @@ -1058,12 +1119,12 @@ void ScaleARGBRowDown2Box_Any_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); -void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, @@ -1098,12 +1159,12 @@ void ScaleARGBRowDownEvenBox_RVV(const uint8_t* src_argb, int src_stepx, uint8_t* dst_argb, int dst_width); -void ScaleARGBRowDownEven_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDownEven_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_ptr, int dst_width); -void ScaleARGBRowDownEvenBox_Any_SSE2(const uint8_t* src_ptr, +void ScaleARGBRowDownEvenBox_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_ptr, @@ -1130,10 +1191,14 @@ void ScaleARGBRowDownEvenBox_Any_LSX(const uint8_t* src_ptr, int dst_width); // UV Row functions -void ScaleUVRowDown2Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_uv, - int dst_width); +void ScaleUVRowDown2_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width); +void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width); void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_uv, @@ -1174,10 +1239,14 @@ void ScaleUVRowDown2Box_RVV(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width); -void ScaleUVRowDown2Box_Any_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width); +void ScaleUVRowDown2_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); +void ScaleUVRowDown2Linear_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width); void ScaleUVRowDown2Box_Any_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -1194,6 +1263,11 @@ void ScaleUVRowDown2Box_Any_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width); +void ScaleUVRowDownEven_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_uv, + int dst_width); void ScaleUVRowDownEven_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, @@ -1209,6 +1283,11 @@ void ScaleUVRowDownEven_RVV(const uint8_t* src_ptr, int32_t src_stepx, uint8_t* dst_uv, int dst_width); +void ScaleUVRowDownEven_Any_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_ptr, + int dst_width); void ScaleUVRowDownEven_Any_NEON(const uint8_t* src_ptr, ptrdiff_t src_stride, int src_stepx, diff --git a/include/libyuv/scale_uv.h b/include/libyuv/scale_uv.h index f6cc8e40f..168c5d949 100644 --- a/include/libyuv/scale_uv.h +++ b/include/libyuv/scale_uv.h @@ -33,7 +33,6 @@ int UVScale(const uint8_t* src_uv, enum FilterMode filtering); // Scale a 16 bit UV image. -// This function is currently incomplete, it can't handle all cases. LIBYUV_API int UVScale_16(const uint16_t* src_uv, int src_stride_uv, diff --git a/include/libyuv/version.h b/include/libyuv/version.h index caf8590aa..5ba507c6d 100644 --- a/include/libyuv/version.h +++ b/include/libyuv/version.h @@ -11,6 +11,6 @@ #ifndef INCLUDE_LIBYUV_VERSION_H_ #define INCLUDE_LIBYUV_VERSION_H_ -#define LIBYUV_VERSION 1976 +#define LIBYUV_VERSION 1977 #endif // INCLUDE_LIBYUV_VERSION_H_ diff --git a/include/libyuv/video_common.h b/include/libyuv/video_common.h index e60707ae5..ee989f786 100644 --- a/include/libyuv/video_common.h +++ b/include/libyuv/video_common.h @@ -84,9 +84,9 @@ enum FourCC { FOURCC_24BG = FOURCC('2', '4', 'B', 'G'), FOURCC_RAW = FOURCC('r', 'a', 'w', ' '), FOURCC_RGBA = FOURCC('R', 'G', 'B', 'A'), - FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE. - FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE. - FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE. + FOURCC_RGBP = FOURCC('R', 'G', 'B', 'P'), // rgb565 LE. deprecated. + FOURCC_RGBO = FOURCC('R', 'G', 'B', 'O'), // argb1555 LE. deprecated. + FOURCC_R444 = FOURCC('R', '4', '4', '4'), // argb4444 LE. deprecated. // 1 Primary Compressed YUV format. FOURCC_MJPG = FOURCC('M', 'J', 'P', 'G'), @@ -135,9 +135,9 @@ enum FourCC { FOURCC_BGR3 = FOURCC('B', 'G', 'R', '3'), // Alias for 24BG. FOURCC_CM32 = FOURCC(0, 0, 0, 32), // Alias for BGRA kCMPixelFormat_32ARGB FOURCC_CM24 = FOURCC(0, 0, 0, 24), // Alias for RAW kCMPixelFormat_24RGB - FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO. - FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP. - FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO. + FOURCC_L555 = FOURCC('L', '5', '5', '5'), // Alias for RGBO. deprecated. + FOURCC_L565 = FOURCC('L', '5', '6', '5'), // Alias for RGBP. deprecated. + FOURCC_5551 = FOURCC('5', '5', '5', '1'), // Alias for RGBO. deprecated. // deprecated formats. Not supported, but defined for backward compatibility. FOURCC_I411 = FOURCC('I', '4', '1', '1'), @@ -177,9 +177,9 @@ enum FourCCBpp { FOURCC_BPP_AB64 = 64, FOURCC_BPP_24BG = 24, FOURCC_BPP_RAW = 24, - FOURCC_BPP_RGBP = 16, - FOURCC_BPP_RGBO = 16, - FOURCC_BPP_R444 = 16, + FOURCC_BPP_RGBP = 16, // deprecated + FOURCC_BPP_RGBO = 16, // deprecated + FOURCC_BPP_R444 = 16, // deprecated FOURCC_BPP_RGGB = 8, FOURCC_BPP_BGGR = 8, FOURCC_BPP_GRBG = 8, diff --git a/source/compare.cc b/source/compare.cc index 352cedb1b..c5ef00699 100644 --- a/source/compare.cc +++ b/source/compare.cc @@ -87,34 +87,6 @@ uint32_t HashDjb2(const uint8_t* src, uint64_t count, uint32_t seed) { return seed; } -static uint32_t ARGBDetectRow_C(const uint8_t* argb, int width) { - int x; - for (x = 0; x < width - 1; x += 2) { - if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. - return FOURCC_BGRA; - } - if (argb[3] != 255) { // Fourth byte is not Alpha of 255, so not BGRA. - return FOURCC_ARGB; - } - if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255. - return FOURCC_BGRA; - } - if (argb[7] != 255) { // Second pixel fourth byte is not Alpha of 255. - return FOURCC_ARGB; - } - argb += 8; - } - if (width & 1) { - if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. - return FOURCC_BGRA; - } - if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA. - return FOURCC_ARGB; - } - } - return 0; -} - // Scan an opaque argb image and return fourcc based on alpha offset. // Returns FOURCC_ARGB, FOURCC_BGRA, or 0 if unknown. LIBYUV_API @@ -122,6 +94,7 @@ uint32_t ARGBDetect(const uint8_t* argb, int stride_argb, int width, int height) { + uint32_t (*ARGBDetectRow)(const uint8_t* argb, int width) = ARGBDetectRow_C; uint32_t fourcc = 0; int h; @@ -134,8 +107,13 @@ uint32_t ARGBDetect(const uint8_t* argb, height = 1; stride_argb = 0; } +#if defined(HAS_ARGBDETECTROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBDetectRow = ARGBDetectRow_AVX2; + } +#endif for (h = 0; h < height && fourcc == 0; ++h) { - fourcc = ARGBDetectRow_C(argb, width); + fourcc = ARGBDetectRow(argb, width); argb += stride_argb; } return fourcc; @@ -262,6 +240,12 @@ uint64_t ComputeSumSquareError(const uint8_t* src_a, simd_size = 1; // Any count. } #endif +#if defined(HAS_SUMSQUAREERROR_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + SumSquareError = SumSquareError_AVX512VNNI; + simd_size = 1; // Any count. + } +#endif #if defined(HAS_SUMSQUAREERROR_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { SumSquareError = SumSquareError_WASMSIMD; @@ -380,10 +364,10 @@ double I420Psnr(const uint8_t* src_y_a, return SumSquareErrorToPsnr(sse, samples); } -static float Ssim8x8_C(const uint8_t* src_a, - int stride_a, - const uint8_t* src_b, - int stride_b) { +static float Ssim8x8(const uint8_t* src_a, + int stride_a, + const uint8_t* src_b, + int stride_b) { int32_t sum_a = 0; int32_t sum_b = 0; int32_t sum_sq_ab = 0; @@ -615,7 +599,7 @@ double CalcFrameSsim(const uint8_t* src_a, const uint8_t* b = src_b + (ptrdiff_t)i * stride_b; int j; for (j = 0; j < width - 8; j += 4) { - ssim_total += Ssim8x8_C(a + j, stride_a, b + j, stride_b); + ssim_total += Ssim8x8(a + j, stride_a, b + j, stride_b); samples++; } } diff --git a/source/compare_common.cc b/source/compare_common.cc index cd667fdd2..f50bc4708 100644 --- a/source/compare_common.cc +++ b/source/compare_common.cc @@ -12,6 +12,8 @@ #include +#include "libyuv/video_common.h" + #ifdef __cplusplus namespace libyuv { extern "C" { @@ -169,6 +171,34 @@ float SsimWindowRow_C(const uint32_t* row0, return ssim_total; } +uint32_t ARGBDetectRow_C(const uint8_t* argb, int width) { + int x; + for (x = 0; x < width - 1; x += 2) { + if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. + return FOURCC_BGRA; + } + if (argb[3] != 255) { // Fourth byte is not Alpha of 255, so not BGRA. + return FOURCC_ARGB; + } + if (argb[4] != 255) { // Second pixel first byte is not Alpha of 255. + return FOURCC_BGRA; + } + if (argb[7] != 255) { // Second pixel fourth byte is not Alpha of 255. + return FOURCC_ARGB; + } + argb += 8; + } + if (width & 1) { + if (argb[0] != 255) { // First byte is not Alpha of 255, so not ARGB. + return FOURCC_BGRA; + } + if (argb[3] != 255) { // 4th byte is not Alpha of 255, so not BGRA. + return FOURCC_ARGB; + } + } + return 0; +} + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/compare_gcc.cc b/source/compare_gcc.cc index 123d92fea..4ddaa12f9 100644 --- a/source/compare_gcc.cc +++ b/source/compare_gcc.cc @@ -10,6 +10,7 @@ #include "libyuv/compare_row.h" #include "libyuv/row.h" +#include "libyuv/video_common.h" #ifdef __cplusplus namespace libyuv { @@ -23,13 +24,15 @@ extern "C" { // asm statement, and other OS ABIs (e.g. Windows) differ in which registers // are callee-saved. List "memory" first, then "cc", then registers. -// This module is for GCC and clang x64. 32 bit x86 uses compare_win.cc. -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) && \ +// This module is for GCC and clang x86 and x64. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ (!defined(_MSC_VER) || defined(__clang__)) // "memory" clobber prevents the reads from being removed +#ifdef HAS_HAMMINGDISTANCE_SSE42 uint32_t HammingDistance_SSE42(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -77,11 +80,16 @@ uint32_t HammingDistance_SSE42(const uint8_t* src_a, return (uint32_t)(diff); } +#endif // HAS_HAMMINGDISTANCE_SSE42 +#if defined(HAS_HAMMINGDISTANCE_SSSE3) || defined(HAS_HAMMINGDISTANCE_AVX2) || \ + defined(HAS_HAMMINGDISTANCE_AVX512BW) static const vec8 kNibbleMask = {15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}; static const vec8 kBitCount = {0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4}; +#endif +#ifdef HAS_HAMMINGDISTANCE_SSSE3 uint32_t HammingDistance_SSSE3(const uint8_t* src_a, const uint8_t* src_b, int count) { @@ -141,6 +149,7 @@ uint32_t HammingDistance_SSSE3(const uint8_t* src_a, return diff; } +#endif // HAS_HAMMINGDISTANCE_SSSE3 #ifdef HAS_HAMMINGDISTANCE_AVX2 uint32_t HammingDistance_AVX2(const uint8_t* src_a, @@ -507,11 +516,78 @@ uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count), [sse] "=r"(sse) : [mask] "m"(mask) - : "cc", "memory", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); return sse; } #endif // HAS_SUMSQUAREERROR_AVX512BW +#ifdef HAS_SUMSQUAREERROR_AVX512VNNI +// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked +// loads that read zeros, so any count is supported. +uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + uint32_t sse; + const uint64_t mask = LowBytesMask64(count & 63); + asm volatile( + "vpxord %%zmm0,%%zmm0,%%zmm0 \n" + "vpxord %%zmm4,%%zmm4,%%zmm4 \n" + "vpxord %%zmm5,%%zmm5,%%zmm5 \n" + "kmovq %[mask],%%k1 \n" + "sub %[src_a],%[src_b] \n" + "sub $0x40,%[count] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%[src_a]),%%zmm1 \n" + "vmovdqu8 (%[src_a],%[src_b]),%%zmm2 \n" + "add $0x40,%[src_a] \n" + "vpsubusb %%zmm2,%%zmm1,%%zmm3 \n" // abs difference trick + "vpsubusb %%zmm1,%%zmm2,%%zmm2 \n" + "vporq %%zmm3,%%zmm2,%%zmm1 \n" + "vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n" // u16 + "vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n" + "vpdpwssd %%zmm2,%%zmm2,%%zmm0 \n" // square + hadd + acc + "vpdpwssd %%zmm1,%%zmm1,%%zmm4 \n" + "sub $0x40,%[count] \n" + "jge 1b \n" + + // Remainder: 1 to 63 bytes, masked loads. + "2: \n" + "add $0x40,%[count] \n" + "je 3f \n" + "vmovdqu8 (%[src_a]),%%zmm1%{%%k1%}%{z%} \n" + "vmovdqu8 (%[src_a],%[src_b]),%%zmm2%{%%k1%}%{z%} \n" + "vpsubusb %%zmm2,%%zmm1,%%zmm3 \n" + "vpsubusb %%zmm1,%%zmm2,%%zmm2 \n" + "vporq %%zmm3,%%zmm2,%%zmm1 \n" + "vpunpcklbw %%zmm5,%%zmm1,%%zmm2 \n" + "vpunpckhbw %%zmm5,%%zmm1,%%zmm1 \n" + "vpdpwssd %%zmm2,%%zmm2,%%zmm0 \n" + "vpdpwssd %%zmm1,%%zmm1,%%zmm4 \n" + + // Add 16 dwords. + "3: \n" + "vpaddd %%zmm4,%%zmm0,%%zmm0 \n" + "vextracti64x4 $1,%%zmm0,%%ymm1 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vextracti128 $1,%%ymm0,%%xmm1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpshufd $0xee,%%xmm0,%%xmm1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpshufd $0x1,%%xmm0,%%xmm1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,%[sse] \n" + "vzeroupper \n" + : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [count] "+r"(count), + [sse] "=r"(sse) + : [mask] "m"(mask) + : "memory", "cc", "k1", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + return sse; +} +#endif // HAS_SUMSQUAREERROR_AVX512VNNI + #if defined(HAS_HASHDJB2_AVX2) || defined(HAS_HASHDJB2_AVX512BW) // HashDjb2 is hash = hash * 33 ^ n + sum(src[i] * 33 ^ (n - 1 - i)). // The 32 bit multipliers 33 ^ (63 - i) are split into 16 bit halves so that @@ -812,10 +888,10 @@ void Ssim4x4Row_SSE2(const uint8_t* src_a, "movdqu %%xmm3,0x20(%[dst]) \n" "movdqu %%xmm4,0x30(%[dst]) \n" "add $0x40,%[dst] \n" - "sub $0x10,%[width] \n" + "subl $0x10,%[width] \n" "jg 1b \n" : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), - [width] "+r"(width) + [width] "+rm"(width) : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); @@ -916,11 +992,11 @@ void Ssim4x4Row_AVX2(const uint8_t* src_a, "vmovdqu %%ymm2,0x40(%[dst]) \n" "vmovdqu %%ymm3,0x60(%[dst]) \n" "add $0x80,%[dst] \n" - "sub $0x20,%[width] \n" + "subl $0x20,%[width] \n" "jg 1b \n" "vzeroupper \n" : [src_a] "+r"(src_a), [src_b] "+r"(src_b), [dst] "+r"(dst), - [width] "+r"(width) + [width] "+rm"(width) : [stride_a] "r"((ptrdiff_t)stride_a), [stride_b] "r"((ptrdiff_t)stride_b) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); @@ -1094,7 +1170,7 @@ float SsimWindowRow_SSE2(const uint32_t* row0, float total; asm volatile( "sub %[row0],%[row1] \n" - "xorps %[total],%[total] \n" + "xorps %%xmm0,%%xmm0 \n" LABELALIGN "1: \n" @@ -1151,19 +1227,20 @@ float SsimWindowRow_SSE2(const uint32_t* row0, "mulps %%xmm5,%%xmm2 \n" "mulps %%xmm3,%%xmm1 \n" "divps %%xmm1,%%xmm2 \n" - "addps %%xmm2,%[total] \n" + "addps %%xmm2,%%xmm0 \n" "sub $0x4,%[num_windows] \n" "jg 1b \n" - "movhlps %[total],%%xmm1 \n" - "addps %%xmm1,%[total] \n" - "movaps %[total],%%xmm1 \n" - "shufps $0x1,%[total],%%xmm1 \n" - "addss %%xmm1,%[total] \n" + "movhlps %%xmm0,%%xmm1 \n" + "addps %%xmm1,%%xmm0 \n" + "movaps %%xmm0,%%xmm1 \n" + "shufps $0x1,%%xmm0,%%xmm1 \n" + "addss %%xmm1,%%xmm0 \n" + "movss %%xmm0,%[total] \n" : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows), - [total] "=&x"(total) + [total] "=m"(total) : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2) - : "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); return total; } #endif // HAS_SSIMWINDOWROW_SSE2 @@ -1176,7 +1253,7 @@ float SsimWindowRow_AVX2(const uint32_t* row0, float total; asm volatile( "sub %[row0],%[row1] \n" - "vxorps %t[total],%t[total],%t[total] \n" + "vxorps %%ymm0,%%ymm0,%%ymm0 \n" LABELALIGN "1: \n" @@ -1231,21 +1308,22 @@ float SsimWindowRow_AVX2(const uint32_t* row0, "vmulps %%ymm3,%%ymm1,%%ymm1 \n" "vmulps %%ymm5,%%ymm4,%%ymm4 \n" "vdivps %%ymm4,%%ymm1,%%ymm1 \n" - "vaddps %%ymm1,%t[total],%t[total] \n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" "sub $0x8,%[num_windows] \n" "jg 1b \n" - "vextractf128 $0x1,%t[total],%%xmm1 \n" - "vaddps %%xmm1,%x[total],%x[total] \n" - "vmovhlps %x[total],%%xmm1,%%xmm1 \n" - "vaddps %%xmm1,%x[total],%x[total] \n" - "vshufps $0x1,%x[total],%x[total],%%xmm1 \n" - "vaddss %%xmm1,%x[total],%x[total] \n" + "vextractf128 $0x1,%%ymm0,%%xmm1 \n" + "vaddps %%xmm1,%%xmm0,%%xmm0 \n" + "vmovhlps %%xmm0,%%xmm1,%%xmm1 \n" + "vaddps %%xmm1,%%xmm0,%%xmm0 \n" + "vshufps $0x1,%%xmm0,%%xmm0,%%xmm1 \n" + "vaddss %%xmm1,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,%[total] \n" "vzeroupper \n" : [row0] "+r"(row0), [row1] "+r"(row1), [num_windows] "+r"(num_windows), - [total] "=&x"(total) + [total] "=m"(total) : [c1] "m"(kSsimC1), [c2] "m"(kSsimC2) - : "memory", "cc", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); return total; } #endif // HAS_SSIMWINDOWROW_AVX2 @@ -1370,8 +1448,51 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, #undef SSIM_WINDOW_AVX512BW #endif // HAS_SSIMWINDOWROW_AVX512BW -#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && - // !defined(LIBYUV_ENABLE_ROWWIN) +#ifdef HAS_ARGBDETECTROW_AVX2 +uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) { + uint32_t diff = 0; + asm volatile( + "vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + LABELALIGN + "1: \n" + "vpcmpeqb (%[argb]),%%ymm1,%%ymm0 \n" + "vpmovmskb %%ymm0,%[diff] \n" + "not %[diff] \n" + "and $0x99999999,%[diff] \n" + "jnz 99f \n" + "add $0x20,%[argb] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + LABELALIGN + "3: \n" + "vmovd (%[argb]),%%xmm0 \n" + "add $0x4,%[argb] \n" + "vpcmpeqb %%xmm1,%%xmm0,%%xmm0 \n" + "vpmovmskb %%xmm0,%[diff] \n" + "not %[diff] \n" + "and $0x9,%[diff] \n" + "jnz 99f \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + "99: \n" + "vzeroupper \n" + : [argb] "+r"(argb), [width] "+r"(width), [diff] "+r"(diff) + : + : "memory", "cc", "xmm0", "xmm1"); + if (diff) { + return (__builtin_ctz(diff) & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB; + } + return 0; +} +#endif // HAS_ARGBDETECTROW_AVX2 + +#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || + // defined(__i386__)) && !defined(LIBYUV_ENABLE_ROWWIN) #ifdef __cplusplus } // extern "C" diff --git a/source/compare_win.cc b/source/compare_win.cc index d1166ad46..b969a3091 100644 --- a/source/compare_win.cc +++ b/source/compare_win.cc @@ -10,14 +10,15 @@ #include "libyuv/compare_row.h" #include "libyuv/row.h" +#include "libyuv/video_common.h" -// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for -// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(_M_IX86) || \ - ((defined(__x86_64__) || defined(_M_X64)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)))) +// This module is for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) #include // For SSE2 to AVX512 intrinsics #include // For memcpy @@ -39,12 +40,15 @@ extern "C" { #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) +#define LIBYUV_TARGET_AVX512VNNI \ + __attribute__((target("avx512vnni,avx512bw,avx512vl,avx512f"))) #else #define LIBYUV_TARGET_SSE2 #define LIBYUV_TARGET_SSE41 #define LIBYUV_TARGET_SSE42 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW +#define LIBYUV_TARGET_AVX512VNNI #endif // defined(__clang__) || defined(__GNUC__) #if defined(HAS_HAMMINGDISTANCE_SSE42) @@ -245,6 +249,44 @@ uint32_t SumSquareError_AVX512BW(const uint8_t* src_a, } #endif // HAS_SUMSQUAREERROR_AVX512BW +#if defined(HAS_SUMSQUAREERROR_AVX512VNNI) +// Process 64 bytes per loop and the remainder of 1 to 63 bytes with masked +// loads that read zeros, so any count is supported. +LIBYUV_TARGET_AVX512VNNI +uint32_t SumSquareError_AVX512VNNI(const uint8_t* src_a, + const uint8_t* src_b, + int count) { + const __m512i zero = _mm512_setzero_si512(); + __m512i sum0 = _mm512_setzero_si512(); + __m512i sum1 = _mm512_setzero_si512(); + while (count >= 64) { + __m512i a = _mm512_loadu_si512(src_a); + __m512i b = _mm512_loadu_si512(src_b); + __m512i d = + _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a)); + __m512i lo = _mm512_unpacklo_epi8(d, zero); + __m512i hi = _mm512_unpackhi_epi8(d, zero); + sum0 = _mm512_dpwssd_epi32(sum0, lo, lo); + sum1 = _mm512_dpwssd_epi32(sum1, hi, hi); + src_a += 64; + src_b += 64; + count -= 64; + } + if (count > 0) { + const __mmask64 k = LowBytesMask64(count); + __m512i a = _mm512_maskz_loadu_epi8(k, src_a); + __m512i b = _mm512_maskz_loadu_epi8(k, src_b); + __m512i d = + _mm512_or_si512(_mm512_subs_epu8(a, b), _mm512_subs_epu8(b, a)); + __m512i lo = _mm512_unpacklo_epi8(d, zero); + __m512i hi = _mm512_unpackhi_epi8(d, zero); + sum0 = _mm512_dpwssd_epi32(sum0, lo, lo); + sum1 = _mm512_dpwssd_epi32(sum1, hi, hi); + } + return (uint32_t)_mm512_reduce_add_epi32(_mm512_add_epi32(sum0, sum1)); +} +#endif // HAS_SUMSQUAREERROR_AVX512VNNI + #if defined(HAS_HASHDJB2_SSE41) // 33 ^ (15 - i) for byte i of 16. static const uint32_t kHashMul16[16] = { @@ -892,6 +934,39 @@ float SsimWindowRow_AVX512BW(const uint32_t* row0, } #endif // HAS_SSIMWINDOWROW_AVX512BW +#if defined(HAS_ARGBDETECTROW_AVX2) +LIBYUV_TARGET_AVX2 +uint32_t ARGBDetectRow_AVX2(const uint8_t* argb, int width) { + if (width >= 8) { + const __m256i mask_rgb = _mm256_set1_epi32(0x00ffff00); + const __m256i ones = _mm256_set1_epi8(-1); + do { + __m256i v = + _mm256_or_si256(_mm256_loadu_si256((const __m256i*)argb), mask_rgb); + uint32_t diff = + ~(uint32_t)_mm256_movemask_epi8(_mm256_cmpeq_epi8(v, ones)); + if (diff) { +#if defined(_MSC_VER) && !defined(__clang__) + unsigned long bit; // NOLINT(runtime/int) + _BitScanForward(&bit, diff); +#else + int bit = __builtin_ctz(diff); +#endif + _mm256_zeroupper(); + return (bit & 3) == 0 ? FOURCC_BGRA : FOURCC_ARGB; + } + argb += 32; + width -= 8; + } while (width >= 8); + _mm256_zeroupper(); + } + if (width > 0) { + return ARGBDetectRow_C(argb, width); + } + return 0; +} +#endif // HAS_ARGBDETECTROW_AVX2 + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/convert.cc b/source/convert.cc index 67385d9ff..50ea99484 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -355,6 +355,13 @@ static int I41xToI420(const uint16_t* src_y, if (TestCpuFlag(kCpuHasRVV)) { HalfWidthRow_16To8 = HalfWidthRow_16To8_RVV; } +#endif + } else { +#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HalfWidthRow_16To8 = HalfWidthRow_16To8_Odd_AVX2; + } #endif } @@ -1387,6 +1394,11 @@ int MT2TToP010(const uint8_t* src_y, if (!row_buf) return 1; +#if defined(HAS_UNPACKMT2T_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + UnpackMT2T = UnpackMT2T_AVX2; + } +#endif #if defined(HAS_UNPACKMT2T_NEON) if (TestCpuFlag(kCpuHasNEON)) { UnpackMT2T = UnpackMT2T_NEON; @@ -2313,6 +2325,13 @@ static int Px10ToI420(const uint16_t* src_y, if (TestCpuFlag(kCpuHasRVV)) { Half16To8 = HalfWidthRow_16To8_RVV; } +#endif + } else { +#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + Half16To8 = HalfWidthRow_16To8_Odd_AVX2; + } #endif } @@ -2516,6 +2535,12 @@ int YUY2ToI420(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_AVX512BW) && defined(HAS_YUY2TOUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToUVRow = YUY2ToUVRow_AVX512BW; + YUY2ToYRow = YUY2ToYRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToYRow = YUY2ToYRow_Any_NEON; @@ -2552,6 +2577,12 @@ int YUY2ToI420(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_HVX) && defined(HAS_YUY2TOUVROW_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + YUY2ToYRow = YUY2ToYRow_HVX; + YUY2ToUVRow = YUY2ToUVRow_HVX; + } +#endif for (y = 0; y < height - 1; y += 2) { YUY2ToUVRow(src_yuy2, src_stride_yuy2, dst_u, dst_v, width); @@ -2616,6 +2647,12 @@ int UYVYToI420(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_AVX512BW) && defined(HAS_UYVYTOUVROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToUVRow = UYVYToUVRow_AVX512BW; + UYVYToYRow = UYVYToYRow_AVX512BW; + } +#endif #if defined(HAS_UYVYTOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { UYVYToYRow = UYVYToYRow_Any_NEON; @@ -2652,6 +2689,12 @@ int UYVYToI420(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_HVX) && defined(HAS_UYVYTOUVROW_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + UYVYToYRow = UYVYToYRow_HVX; + UYVYToUVRow = UYVYToUVRow_HVX; + } +#endif for (y = 0; y < height - 1; y += 2) { UYVYToUVRow(src_uyvy, src_stride_uyvy, dst_u, dst_v, width); @@ -2693,24 +2736,13 @@ int AYUVToNV12(const uint8_t* src_ayuv, src_ayuv = src_ayuv + (ptrdiff_t)(height - 1) * src_stride_ayuv; src_stride_ayuv = -src_stride_ayuv; } -// place holders for future intel code -#if defined(HAS_AYUVTOYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - AYUVToUVRow = AYUVToUVRow_Any_SSE2; - AYUVToYRow = AYUVToYRow_Any_SSE2; - if (IS_ALIGNED(width, 16)) { - AYUVToUVRow = AYUVToUVRow_SSE2; - AYUVToYRow = AYUVToYRow_SSE2; - } - } -#endif -#if defined(HAS_AYUVTOYROW_AVX2) +#if defined(HAS_AYUVTOYROW_AVX2) && defined(HAS_AYUVTOUVROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { - AYUVToUVRow = AYUVToUVRow_Any_AVX2; AYUVToYRow = AYUVToYRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - AYUVToUVRow = AYUVToUVRow_AVX2; + AYUVToUVRow = AYUVToUVRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { AYUVToYRow = AYUVToYRow_AVX2; + AYUVToUVRow = AYUVToUVRow_AVX2; } } #endif @@ -2780,24 +2812,13 @@ int AYUVToNV21(const uint8_t* src_ayuv, src_ayuv = src_ayuv + (ptrdiff_t)(height - 1) * src_stride_ayuv; src_stride_ayuv = -src_stride_ayuv; } -// place holders for future intel code -#if defined(HAS_AYUVTOYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - AYUVToVURow = AYUVToVURow_Any_SSE2; - AYUVToYRow = AYUVToYRow_Any_SSE2; - if (IS_ALIGNED(width, 16)) { - AYUVToVURow = AYUVToVURow_SSE2; - AYUVToYRow = AYUVToYRow_SSE2; - } - } -#endif -#if defined(HAS_AYUVTOYROW_AVX2) +#if defined(HAS_AYUVTOYROW_AVX2) && defined(HAS_AYUVTOVUROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { - AYUVToVURow = AYUVToVURow_Any_AVX2; AYUVToYRow = AYUVToYRow_Any_AVX2; - if (IS_ALIGNED(width, 32)) { - AYUVToVURow = AYUVToVURow_AVX2; + AYUVToVURow = AYUVToVURow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { AYUVToYRow = AYUVToYRow_AVX2; + AYUVToVURow = AYUVToVURow_AVX2; } } #endif @@ -3546,6 +3567,14 @@ static int AR30ToI420Matrix(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + AR30ToARGBRow = AR30ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + AR30ToARGBRow = AR30ToARGBRow_AVX2; + } + } +#endif #if defined(HAS_AR30TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToARGBRow = AR30ToARGBRow_AVX512BW; @@ -3791,12 +3820,6 @@ int AB30ToI420(const uint8_t* src_ab30, &kAbgrI601Constants, width, height); } -// Enabled if 1 pass is available -#if (defined(HAS_RGB24TOYROW_NEON) || defined(HAS_RGB24TOYROW_LSX) || \ - defined(HAS_RGB24TOYROW_RVV)) -#define HAS_RGB24TOYROW -#endif - // Convert RGB24 to I420. LIBYUV_API int RGB24ToI420(const uint8_t* src_rgb24, @@ -4118,12 +4141,6 @@ int RGB24ToJ420(const uint8_t* src_rgb24, return 0; } -// Enabled if 1 pass is available -#if (defined(HAS_RAWTOYROW_NEON) || defined(HAS_RAWTOYROW_LSX) || \ - defined(HAS_RAWTOYROW_RVV)) -#define HAS_RAWTOYROW -#endif - // Convert RAW to I420. LIBYUV_API int RAWToI420(const uint8_t* src_raw, diff --git a/source/convert_argb.cc b/source/convert_argb.cc index e9d8b9c1c..e79dcb8ad 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -631,10 +631,7 @@ int I444ToARGBMatrix(const uint8_t* src_y, #endif #if defined(HAS_I444TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I444ToARGBRow = I444ToARGBRow_AVX512BW; - } + I444ToARGBRow = I444ToARGBRow_AVX512BW; } #endif #if defined(HAS_I444TOARGBROW_NEON) @@ -888,6 +885,11 @@ int I444ToRGB24Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TORGB24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI; + } +#endif #if defined(HAS_I444TORGB24ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToRGB24Row = I444ToRGB24Row_Any_NEON; @@ -906,14 +908,6 @@ int I444ToRGB24Matrix(const uint8_t* src_y, I444ToRGB24Row = I444ToRGB24Row_SME; } #endif -#if defined(HAS_I444TORGB24ROW_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - I444ToRGB24Row = I444ToRGB24Row_Any_LSX; - if (IS_ALIGNED(width, 16)) { - I444ToRGB24Row = I444ToRGB24Row_LSX; - } - } -#endif #if defined(HAS_I444TORGB24ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444ToRGB24Row = I444ToRGB24Row_RVV; @@ -1030,6 +1024,11 @@ int I010ToAR30Matrix(const uint16_t* src_y, I210ToAR30Row = I210ToAR30Row_AVX2; } } +#endif +#if defined(HAS_I210TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210ToAR30Row = I210ToAR30Row_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1192,6 +1191,11 @@ int I012ToAR30Matrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I212TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I212ToAR30Row = I212ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_I212TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I212ToAR30Row = I212ToAR30Row_Any_NEON; @@ -1286,6 +1290,11 @@ int I210ToAR30Matrix(const uint16_t* src_y, I210ToAR30Row = I210ToAR30Row_AVX2; } } +#endif +#if defined(HAS_I210TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210ToAR30Row = I210ToAR30Row_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I210ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1460,6 +1469,11 @@ int I410ToAR30Matrix(const uint16_t* src_y, I410ToAR30Row = I410ToAR30Row_AVX2; } } +#endif +#if defined(HAS_I410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToAR30Row = I410ToAR30Row_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I410ToAR30Row(src_y, src_u, src_v, dst_ar30, yuvconstants, width); @@ -1533,6 +1547,11 @@ int I010ToARGBMatrix(const uint16_t* src_y, I210ToARGBRow = I210ToARGBRow_AVX2; } } +#endif +#if defined(HAS_I210TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210ToARGBRow = I210ToARGBRow_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -1699,6 +1718,11 @@ int I012ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I212TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I212ToARGBRow = I212ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I212TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I212ToARGBRow = I212ToARGBRow_Any_NEON; @@ -1799,6 +1823,11 @@ int I210ToARGBMatrix(const uint16_t* src_y, I210ToARGBRow = I210ToARGBRow_AVX2; } } +#endif +#if defined(HAS_I210TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210ToARGBRow = I210ToARGBRow_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I210ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -1986,6 +2015,11 @@ int I410ToARGBMatrix(const uint16_t* src_y, I410ToARGBRow = I410ToARGBRow_AVX2; } } +#endif +#if defined(HAS_I410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToARGBRow = I410ToARGBRow_AVX512BW; + } #endif for (y = 0; y < height; ++y) { I410ToARGBRow(src_y, src_u, src_v, dst_argb, yuvconstants, width); @@ -2038,6 +2072,11 @@ int P010ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P210TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P210ToARGBRow = P210ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P210TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P210ToARGBRow = P210ToARGBRow_Any_NEON; @@ -2108,6 +2147,11 @@ int P210ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P210TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P210ToARGBRow = P210ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P210TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P210ToARGBRow = P210ToARGBRow_Any_NEON; @@ -2176,6 +2220,11 @@ int P410ToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToARGBRow = P410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P410TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToARGBRow = P410ToARGBRow_Any_NEON; @@ -2244,6 +2293,11 @@ int P010ToAR30Matrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P210TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P210ToAR30Row = P210ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P210TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P210ToAR30Row = P210ToAR30Row_Any_NEON; @@ -2314,6 +2368,11 @@ int P210ToAR30Matrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P210TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P210ToAR30Row = P210ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P210TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P210ToAR30Row = P210ToAR30Row_Any_NEON; @@ -2382,6 +2441,11 @@ int P410ToAR30Matrix(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToAR30Row = P410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P410TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToAR30Row = P410ToAR30Row_Any_NEON; @@ -2822,6 +2886,11 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I444ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I444ALPHATOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON; @@ -3119,6 +3188,11 @@ int I010AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I210ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -3261,6 +3335,11 @@ int I210AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I210ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I210AlphaToARGBRow = I210AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -3401,6 +3480,11 @@ int I410AlphaToARGBMatrix(const uint16_t* src_y, } } #endif +#if defined(HAS_I410ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -4214,11 +4298,11 @@ int RAWToRGBA(const uint8_t* src_raw, height = 1; src_stride_raw = dst_stride_rgba = 0; } -#if defined(HAS_RAWTORGBAROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - RAWToRGBARow = RAWToRGBARow_Any_SSSE3; +#if defined(HAS_RAWTORGBAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RAWToRGBARow = RAWToRGBARow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - RAWToRGBARow = RAWToRGBARow_SSSE3; + RAWToRGBARow = RAWToRGBARow_AVX2; } } #endif @@ -4370,6 +4454,11 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555, } } #endif +#if defined(HAS_ARGB1555TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGB1555ToARGBRow = ARGB1555ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGB1555TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_NEON; @@ -4530,6 +4619,14 @@ int AR30ToARGB(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOARGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + AR30ToARGBRow = AR30ToARGBRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + AR30ToARGBRow = AR30ToARGBRow_AVX2; + } + } +#endif #if defined(HAS_AR30TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToARGBRow = AR30ToARGBRow_AVX512BW; @@ -4579,6 +4676,14 @@ int AR30ToABGR(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOABGRROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + AR30ToABGRRow = AR30ToABGRRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + AR30ToABGRRow = AR30ToABGRRow_AVX2; + } + } +#endif #if defined(HAS_AR30TOABGRROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToABGRRow = AR30ToABGRRow_AVX512BW; @@ -4628,6 +4733,14 @@ int AR30ToAB30(const uint8_t* src_ar30, } } #endif +#if defined(HAS_AR30TOAB30ROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + AR30ToAB30Row = AR30ToAB30Row_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + AR30ToAB30Row = AR30ToAB30Row_AVX2; + } + } +#endif #if defined(HAS_AR30TOAB30ROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { AR30ToAB30Row = AR30ToAB30Row_AVX512BW; @@ -5391,6 +5504,11 @@ int YUY2ToARGBMatrix(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToARGBRow = YUY2ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToARGBRow = YUY2ToARGBRow_Any_NEON; @@ -5491,6 +5609,11 @@ int UYVYToARGBMatrix(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToARGBRow = UYVYToARGBRow_AVX512BW; + } +#endif #if defined(HAS_UYVYTOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { UYVYToARGBRow = UYVYToARGBRow_Any_NEON; @@ -5721,6 +5844,11 @@ int I422ToRGBAMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGBAROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I422ToRGBARow = I422ToRGBARow_AVX512BW; + } +#endif #if defined(HAS_I422TORGBAROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToRGBARow = I422ToRGBARow_Any_NEON; @@ -5958,6 +6086,11 @@ int I420ToRGBAMatrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGBAROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I422ToRGBARow = I422ToRGBARow_AVX512BW; + } +#endif #if defined(HAS_I422TORGBAROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToRGBARow = I422ToRGBARow_Any_NEON; @@ -6474,6 +6607,12 @@ int I420ToARGB1555(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOARGB1555ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { + I422ToARGB1555Row = I422ToARGB1555Row_AVX512BW; + } +#endif #if defined(HAS_I422TOARGB1555ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToARGB1555Row = I422ToARGB1555Row_Any_NEON; @@ -6573,6 +6712,12 @@ int I420ToARGB4444(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TOARGB4444ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { + I422ToARGB4444Row = I422ToARGB4444Row_AVX512BW; + } +#endif #if defined(HAS_I422TOARGB4444ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToARGB4444Row = I422ToARGB4444Row_Any_NEON; @@ -6674,6 +6819,12 @@ int I420ToRGB565Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB565ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { + I422ToRGB565Row = I422ToRGB565Row_AVX512BW; + } +#endif #if defined(HAS_I422TORGB565ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToRGB565Row = I422ToRGB565Row_Any_NEON; @@ -6833,6 +6984,12 @@ int I422ToRGB565Matrix(const uint8_t* src_y, } } #endif +#if defined(HAS_I422TORGB565ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW | kCpuHasAVX512VL) == + (kCpuHasAVX512BW | kCpuHasAVX512VL)) { + I422ToRGB565Row = I422ToRGB565Row_AVX512BW; + } +#endif #if defined(HAS_I422TORGB565ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I422ToRGB565Row = I422ToRGB565Row_Any_NEON; @@ -7018,14 +7175,6 @@ int I420ToRGB565Dither(const uint8_t* src_y, I422ToARGBRow = I422ToARGBRow_HVX; } #endif -#if defined(HAS_ARGBTORGB565DITHERROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_SSE2; - if (IS_ALIGNED(width, 4)) { - ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_SSE2; - } - } -#endif #if defined(HAS_ARGBTORGB565DITHERROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_AVX2; @@ -7034,6 +7183,11 @@ int I420ToRGB565Dither(const uint8_t* src_y, } } #endif +#if defined(HAS_ARGBTORGB565DITHERROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX512BW; + } +#endif #if defined(HAS_ARGBTORGB565DITHERROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON; @@ -7306,10 +7460,7 @@ static int I420ToARGBMatrixBilinear(const uint8_t* src_y, #endif #if defined(HAS_I444TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I444ToARGBRow = I444ToARGBRow_AVX512BW; - } + I444ToARGBRow = I444ToARGBRow_AVX512BW; } #endif #if defined(HAS_I444TOARGBROW_NEON) @@ -7477,10 +7628,7 @@ static int I422ToARGBMatrixLinear(const uint8_t* src_y, #endif #if defined(HAS_I444TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I444ToARGBRow = I444ToARGBRow_AVX512BW; - } + I444ToARGBRow = I444ToARGBRow_AVX512BW; } #endif #if defined(HAS_I444TOARGBROW_NEON) @@ -7623,6 +7771,11 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TORGB24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI; + } +#endif #if defined(HAS_I444TORGB24ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToRGB24Row = I444ToRGB24Row_Any_NEON; @@ -7641,14 +7794,6 @@ static int I420ToRGB24MatrixBilinear(const uint8_t* src_y, I444ToRGB24Row = I444ToRGB24Row_SME; } #endif -#if defined(HAS_I444TORGB24ROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - I444ToRGB24Row = I444ToRGB24Row_Any_LASX; - if (IS_ALIGNED(width, 32)) { - I444ToRGB24Row = I444ToRGB24Row_LASX; - } - } -#endif #if defined(HAS_I444TORGB24ROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444ToRGB24Row = I444ToRGB24Row_RVV; @@ -7794,6 +7939,11 @@ static int I010ToAR30MatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToAR30Row = I410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -7919,6 +8069,11 @@ static int I210ToAR30MatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToAR30Row = I410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8023,6 +8178,11 @@ static int I010ToARGBMatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToARGBRow = I410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_SCALEROWUP2_BILINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8147,6 +8307,11 @@ static int I210ToARGBMatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410ToARGBRow = I410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_SCALEROWUP2_LINEAR_12_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { @@ -8241,6 +8406,11 @@ static int I420AlphaToARGBMatrixBilinear( } } #endif +#if defined(HAS_I444ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I444ALPHATOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON; @@ -8259,14 +8429,6 @@ static int I420AlphaToARGBMatrixBilinear( I444AlphaToARGBRow = I444AlphaToARGBRow_SME; } #endif -#if defined(HAS_I444ALPHATOARGBROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - I444AlphaToARGBRow = I444AlphaToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { - I444AlphaToARGBRow = I444AlphaToARGBRow_LASX; - } - } -#endif #if defined(HAS_I444ALPHATOARGBROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444AlphaToARGBRow = I444AlphaToARGBRow_RVV; @@ -8476,6 +8638,11 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, } } #endif +#if defined(HAS_I444ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I444AlphaToARGBRow = I444AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_I444ALPHATOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444AlphaToARGBRow = I444AlphaToARGBRow_Any_NEON; @@ -8494,14 +8661,6 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, I444AlphaToARGBRow = I444AlphaToARGBRow_SME; } #endif -#if defined(HAS_I444ALPHATOARGBROW_LASX) - if (TestCpuFlag(kCpuHasLASX)) { - I444AlphaToARGBRow = I444AlphaToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { - I444AlphaToARGBRow = I444AlphaToARGBRow_LASX; - } - } -#endif #if defined(HAS_I444ALPHATOARGBROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444AlphaToARGBRow = I444AlphaToARGBRow_RVV; @@ -8694,6 +8853,11 @@ static int I010AlphaToARGBMatrixBilinear( } } #endif +#if defined(HAS_I410ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -8898,6 +9062,11 @@ static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_I410ALPHATOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + I410AlphaToARGBRow = I410AlphaToARGBRow_AVX512BW; + } +#endif #if defined(HAS_ARGBATTENUATEROW_SSSE3) if (TestCpuFlag(kCpuHasSSSE3)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_SSSE3; @@ -9039,6 +9208,11 @@ static int P010ToARGBMatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToARGBRow = P410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P410TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToARGBRow = P410ToARGBRow_Any_NEON; @@ -9151,6 +9325,11 @@ static int P210ToARGBMatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOARGBROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToARGBRow = P410ToARGBRow_AVX512BW; + } +#endif #if defined(HAS_P410TOARGBROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToARGBRow = P410ToARGBRow_Any_NEON; @@ -9249,6 +9428,11 @@ static int P010ToAR30MatrixBilinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToAR30Row = P410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P410TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToAR30Row = P410ToAR30Row_Any_NEON; @@ -9361,6 +9545,11 @@ static int P210ToAR30MatrixLinear(const uint16_t* src_y, } } #endif +#if defined(HAS_P410TOAR30ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + P410ToAR30Row = P410ToAR30Row_AVX512BW; + } +#endif #if defined(HAS_P410TOAR30ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { P410ToAR30Row = P410ToAR30Row_Any_NEON; @@ -9461,6 +9650,11 @@ static int I422ToRGB24MatrixLinear(const uint8_t* src_y, } } #endif +#if defined(HAS_I444TORGB24ROW_AVX512VBMI) + if (TestCpuFlag(kCpuHasAVX512VBMI)) { + I444ToRGB24Row = I444ToRGB24Row_AVX512VBMI; + } +#endif #if defined(HAS_I444TORGB24ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { I444ToRGB24Row = I444ToRGB24Row_Any_NEON; diff --git a/source/convert_from.cc b/source/convert_from.cc index 4bca7602f..c4f35cfea 100644 --- a/source/convert_from.cc +++ b/source/convert_from.cc @@ -820,6 +820,11 @@ static int I4xxToPxxx(const uint8_t* src_y, } } #endif +#if defined(HAS_MERGEUVROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MergeUVRow_16 = MergeUVRow_16_AVX512BW; + } +#endif #if defined(HAS_MERGEUVROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { MergeUVRow_16 = MergeUVRow_16_Any_NEON; diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 2337d54f7..cc7691314 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -4040,6 +4040,11 @@ int ARGBToRGB565Dither(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTORGB565DITHERROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_AVX512BW; + } +#endif #if defined(HAS_ARGBTORGB565DITHERROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_NEON; @@ -4123,6 +4128,11 @@ int ARGBToRGB565(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTORGB565ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + ARGBToRGB565Row = ARGBToRGB565Row_AVX512BW; + } +#endif #if defined(HAS_ARGBTORGB565ROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ARGBToRGB565Row = ARGBToRGB565Row_Any_NEON; diff --git a/source/convert_to_argb.cc b/source/convert_to_argb.cc index bc699b647..d3b55687a 100644 --- a/source/convert_to_argb.cc +++ b/source/convert_to_argb.cc @@ -174,10 +174,7 @@ static int NV24ToARGB(const uint8_t* src_y, #endif #if defined(HAS_I444TOARGBROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - I444ToARGBRow = I444ToARGBRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - I444ToARGBRow = I444ToARGBRow_AVX512BW; - } + I444ToARGBRow = I444ToARGBRow_AVX512BW; } #endif #if defined(HAS_I444TOARGBROW_NEON) diff --git a/source/convert_to_i420.cc b/source/convert_to_i420.cc index 67f78b170..0d80d0c0d 100644 --- a/source/convert_to_i420.cc +++ b/source/convert_to_i420.cc @@ -26,16 +26,6 @@ namespace libyuv { extern "C" { #endif -static void HalfRow_C(const uint8_t* src_uv, - ptrdiff_t src_uv_stride, - uint8_t* dst_uv, - int width) { - int x; - for (x = 0; x < width; ++x) { - dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1; - } -} - static int NV16ToI420(const uint8_t* src_y, int src_stride_y, const uint8_t* src_uv, @@ -227,6 +217,8 @@ static int NV24ToI420(const uint8_t* src_y, int x, y; int halfwidth = (width + 1) >> 1; int halfheight = (height + 1) >> 1; + void (*HalfRow)(const uint8_t* src_uv, ptrdiff_t src_uv_stride, + uint8_t* dst_uv, int width) = HalfRow_C; if ((!src_y && dst_y) || !src_uv || !dst_u || !dst_v || width <= 0 || height == 0 || height == INT_MIN) { return -1; @@ -242,6 +234,14 @@ static int NV24ToI420(const uint8_t* src_y, if (dst_y) { CopyPlane(src_y, src_stride_y, dst_y, dst_stride_y, width, height); } +#if defined(HAS_HALFROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HalfRow = HalfRow_Any_AVX2; + if (IS_ALIGNED(width * 2, 32)) { + HalfRow = HalfRow_AVX2; + } + } +#endif align_buffer_64(row_uv, width * 2); if (!row_uv) { @@ -249,7 +249,7 @@ static int NV24ToI420(const uint8_t* src_y, } for (y = 0; y < halfheight; ++y) { if (y * 2 + 1 < height) { - HalfRow_C(src_uv, src_stride_uv, row_uv, width * 2); + HalfRow(src_uv, src_stride_uv, row_uv, width * 2); } else { memcpy(row_uv, src_uv, (size_t)width * 2); } diff --git a/source/cpu_id.cc b/source/cpu_id.cc index 7bfb66de2..f78283a97 100644 --- a/source/cpu_id.cc +++ b/source/cpu_id.cc @@ -405,6 +405,7 @@ static SAFEBUFFERS int GetCpuFlags(void) { int cpu_info1[4] = {0, 0, 0, 0}; int cpu_info7[4] = {0, 0, 0, 0}; int cpu_einfo7[4] = {0, 0, 0, 0}; + int cpu_einfo1e[4] = {0, 0, 0, 0}; int cpu_info24[4] = {0, 0, 0, 0}; int cpu_info21[4] = {0, 0, 0, 0}; int cpu_amdinfo21[4] = {0, 0, 0, 0}; @@ -425,6 +426,9 @@ static SAFEBUFFERS int GetCpuFlags(void) { if ((unsigned)cpu_extinfo0[0] >= 0x80000021u) { CpuId(0x80000021, 0, cpu_amdinfo21); } + if (cpu_info0[0] >= 0x1e) { + CpuId(0x1e, 1, cpu_einfo1e); + } if (cpu_info0[0] >= 0x21) { CpuId(0x21, 0, cpu_info21); } @@ -459,7 +463,8 @@ static SAFEBUFFERS int GetCpuFlags(void) { ((cpu_info7[2] & 0x00001000) ? kCpuHasAVX512VBITALG : 0) | ((cpu_einfo7[3] & 0x00080000) ? kCpuHasAVX10 : 0) | ((cpu_info7[3] & 0x02000000) ? kCpuHasAMXINT8 : 0) | - ((cpu_info21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0); + ((cpu_info21[0] & 0x00800000) ? kCpuHasAVX512BMM : 0) | + ((cpu_einfo1e[0] & 0x00000040) ? kCpuHasAMXTF32 : 0); if (cpu_info0[0] >= 0x24 && (cpu_einfo7[3] & 0x00080000)) { cpu_info |= ((cpu_info24[1] & 0xFF) >= 2) ? kCpuHasAVX10_2 : 0; } diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 1521e3f05..0c8a42542 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -148,6 +148,11 @@ void CopyPlane_16(const uint16_t* src_y, if (src_y == dst_y && src_stride_y == dst_stride_y) { return; } +#if defined(HAS_COPYROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_16_AVX2 : CopyRow_16_Any_AVX2; + } +#endif #if defined(HAS_COPYROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { CopyRow = IS_ALIGNED(width, 16) ? CopyRow_16_NEON : CopyRow_16_Any_NEON; @@ -406,6 +411,11 @@ void Convert8To8Plane(const uint8_t* src_y, } } #endif +#if defined(HAS_CONVERT8TO8ROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + Convert8To8Row = Convert8To8Row_AVX512BW; + } +#endif #if defined(HAS_CONVERT8TO8ROW_HVX) if (TestCpuFlag(kCpuHasHVX)) { Convert8To8Row = Convert8To8Row_HVX; @@ -1009,6 +1019,11 @@ void MergeUVPlane_16(const uint16_t* src_u, } } #endif +#if defined(HAS_MERGEUVROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + MergeUVRow_16 = MergeUVRow_16_AVX512BW; + } +#endif #if defined(HAS_MERGEUVROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { MergeUVRow_16 = MergeUVRow_16_Any_NEON; @@ -1160,6 +1175,11 @@ void ConvertToLSBPlane_16(const uint16_t* src_y, } } #endif +#if defined(HAS_DIVIDEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + DivideRow = DivideRow_16_AVX512BW; + } +#endif #if defined(HAS_DIVIDEROW_16_NEON) if (TestCpuFlag(kCpuHasNEON)) { DivideRow = DivideRow_16_Any_NEON; @@ -1337,11 +1357,11 @@ int DetilePlane(const uint8_t* src_y, dst_stride_y = -dst_stride_y; } -#if defined(HAS_DETILEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - DetileRow = DetileRow_Any_SSE2; +#if defined(HAS_DETILEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + DetileRow = DetileRow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - DetileRow = DetileRow_SSE2; + DetileRow = DetileRow_AVX2; } } #endif @@ -1394,19 +1414,11 @@ int DetilePlane_16(const uint16_t* src_y, dst_stride_y = -dst_stride_y; } -#if defined(HAS_DETILEROW_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - DetileRow_16 = DetileRow_16_Any_SSE2; +#if defined(HAS_DETILEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + DetileRow_16 = DetileRow_16_Any_AVX2; if (IS_ALIGNED(width, 16)) { - DetileRow_16 = DetileRow_16_SSE2; - } - } -#endif -#if defined(HAS_DETILEROW_16_AVX) - if (TestCpuFlag(kCpuHasAVX)) { - DetileRow_16 = DetileRow_16_Any_AVX; - if (IS_ALIGNED(width, 16)) { - DetileRow_16 = DetileRow_16_AVX; + DetileRow_16 = DetileRow_16_AVX2; } } #endif @@ -1466,11 +1478,11 @@ void DetileSplitUVPlane(const uint8_t* src_uv, dst_stride_v = -dst_stride_v; } -#if defined(HAS_DETILESPLITUVROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - DetileSplitUVRow = DetileSplitUVRow_Any_SSSE3; +#if defined(HAS_DETILESPLITUVROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + DetileSplitUVRow = DetileSplitUVRow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - DetileSplitUVRow = DetileSplitUVRow_SSSE3; + DetileSplitUVRow = DetileSplitUVRow_AVX2; } } #endif @@ -1540,11 +1552,11 @@ void DetileToYUY2(const uint8_t* src_y, } #endif -#if defined(HAS_DETILETOYUY2_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - DetileToYUY2 = DetileToYUY2_Any_SSE2; - if (IS_ALIGNED(width, 16)) { - DetileToYUY2 = DetileToYUY2_SSE2; +#if defined(HAS_DETILETOYUY2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + DetileToYUY2 = DetileToYUY2_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + DetileToYUY2 = DetileToYUY2_AVX2; } } #endif @@ -1684,11 +1696,11 @@ void MergeRGBPlane(const uint8_t* src_r, height = 1; src_stride_r = src_stride_g = src_stride_b = dst_stride_rgb = 0; } -#if defined(HAS_MERGERGBROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - MergeRGBRow = MergeRGBRow_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - MergeRGBRow = MergeRGBRow_SSSE3; +#if defined(HAS_MERGERGBROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + MergeRGBRow = MergeRGBRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + MergeRGBRow = MergeRGBRow_AVX2; } } #endif @@ -2536,6 +2548,11 @@ int YUY2ToI422(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToYRow = YUY2ToYRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToYRow = YUY2ToYRow_Any_NEON; @@ -2644,6 +2661,11 @@ int UYVYToI422(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToYRow = UYVYToYRow_AVX512BW; + } +#endif #if defined(HAS_UYVYTOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { UYVYToYRow = UYVYToYRow_Any_NEON; @@ -2740,6 +2762,11 @@ int YUY2ToY(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToYRow = YUY2ToYRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToYRow = YUY2ToYRow_Any_NEON; @@ -2810,6 +2837,11 @@ int UYVYToY(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + UYVYToYRow = UYVYToYRow_AVX512BW; + } +#endif #if defined(HAS_UYVYTOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { UYVYToYRow = UYVYToYRow_Any_NEON; @@ -2899,10 +2931,7 @@ void MirrorPlane(const uint8_t* src_y, #endif #if defined(HAS_MIRRORROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MirrorRow = MirrorRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - MirrorRow = MirrorRow_AVX512BW; - } + MirrorRow = MirrorRow_AVX512BW; } #endif #if defined(HAS_MIRRORROW_LSX) @@ -3095,17 +3124,9 @@ void MirrorUVPlane(const uint8_t* src_uv, } } #endif -#if defined(HAS_MIRRORROW_16_AVX512BW) +#if defined(HAS_MIRRORUVROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MirrorUVRow = reinterpret_cast( - MirrorRow_16_AVX512BW); - } -#elif defined(HAS_MIRRORUVROW_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW)) { - MirrorUVRow = MirrorUVRow_Any_AVX512BW; - if (IS_ALIGNED(width, 32)) { - MirrorUVRow = MirrorUVRow_AVX512BW; - } + MirrorUVRow = MirrorUVRow_AVX512BW; } #endif #if defined(HAS_MIRRORUVROW_LSX) @@ -3447,9 +3468,9 @@ int ARGBBlend(const uint8_t* src_argb0, height = 1; src_stride_argb0 = src_stride_argb1 = dst_stride_argb = 0; } -#if defined(HAS_ARGBBLENDROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ARGBBlendRow = ARGBBlendRow_SSSE3; +#if defined(HAS_ARGBBLENDROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBBlendRow = ARGBBlendRow_AVX2; } #endif #if defined(HAS_ARGBBLENDROW_AVX512BW) @@ -4429,11 +4450,11 @@ void SetPlane(uint8_t* dst_y, } } #endif -#if defined(HAS_SETROW_X86) - if (TestCpuFlag(kCpuHasX86)) { - SetRow = SetRow_Any_X86; - if (IS_ALIGNED(width, 4)) { - SetRow = SetRow_X86; +#if defined(HAS_SETROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SetRow = SetRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + SetRow = SetRow_AVX2; } } #endif @@ -4541,9 +4562,12 @@ int ARGBRect(uint8_t* dst_argb, } } #endif -#if defined(HAS_ARGBSETROW_X86) - if (TestCpuFlag(kCpuHasX86)) { - ARGBSetRow = ARGBSetRow_X86; +#if defined(HAS_ARGBSETROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBSetRow = ARGBSetRow_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + ARGBSetRow = ARGBSetRow_AVX2; + } } #endif #if defined(HAS_ARGBSETROW_LSX) @@ -4771,9 +4795,9 @@ int ARGBGrayTo(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } -#if defined(HAS_ARGBGRAYROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_SSSE3; +#if defined(HAS_ARGBGRAYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBGrayRow = ARGBGrayRow_AVX2; } #endif #if defined(HAS_ARGBGRAYROW_AVX512BW) @@ -4836,9 +4860,9 @@ int ARGBGray(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_ARGBGRAYROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { - ARGBGrayRow = ARGBGrayRow_SSSE3; +#if defined(HAS_ARGBGRAYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBGrayRow = ARGBGrayRow_AVX2; } #endif #if defined(HAS_ARGBGRAYROW_AVX512BW) @@ -4899,9 +4923,14 @@ int ARGBSepia(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_ARGBSEPIAROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { - ARGBSepiaRow = ARGBSepiaRow_SSSE3; +#if defined(HAS_ARGBSEPIAROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBSepiaRow = ARGBSepiaRow_AVX2; + } +#endif +#if defined(HAS_ARGBSEPIAROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBSepiaRow = ARGBSepiaRow_AVX512VNNI; } #endif #if defined(HAS_ARGBSEPIAROW_NEON) @@ -4962,9 +4991,14 @@ int ARGBColorMatrix(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } -#if defined(HAS_ARGBCOLORMATRIXROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 8)) { - ARGBColorMatrixRow = ARGBColorMatrixRow_SSSE3; +#if defined(HAS_ARGBCOLORMATRIXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_AVX2; + } +#endif +#if defined(HAS_ARGBCOLORMATRIXROW_AVX512VNNI) + if (TestCpuFlag(kCpuHasAVX512VNNI)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_AVX512VNNI; } #endif #if defined(HAS_ARGBCOLORMATRIXROW_NEON) @@ -5056,9 +5090,9 @@ int ARGBColorTable(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_ARGBCOLORTABLEROW_X86) - if (TestCpuFlag(kCpuHasX86)) { - ARGBColorTableRow = ARGBColorTableRow_X86; +#if defined(HAS_ARGBCOLORTABLEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBColorTableRow = ARGBColorTableRow_AVX2; } #endif #if defined(HAS_ARGBCOLORTABLEROW_NEON) @@ -5097,9 +5131,9 @@ int RGBColorTable(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_RGBCOLORTABLEROW_X86) - if (TestCpuFlag(kCpuHasX86)) { - RGBColorTableRow = RGBColorTableRow_X86; +#if defined(HAS_RGBCOLORTABLEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + RGBColorTableRow = RGBColorTableRow_AVX2; } #endif #if defined(HAS_RGBCOLORTABLEROW_NEON) @@ -5147,9 +5181,9 @@ int ARGBQuantize(uint8_t* dst_argb, height = 1; dst_stride_argb = 0; } -#if defined(HAS_ARGBQUANTIZEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) { - ARGBQuantizeRow = ARGBQuantizeRow_SSE2; +#if defined(HAS_ARGBQUANTIZEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBQuantizeRow = ARGBQuantizeRow_AVX2; } #endif #if defined(HAS_ARGBQUANTIZEROW_NEON) @@ -5191,11 +5225,6 @@ int ARGBComputeCumulativeSum(const uint8_t* src_argb, if (!dst_cumsum || !src_argb || width <= 0 || height <= 0) { return -1; } -#if defined(HAS_COMPUTECUMULATIVESUMROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2; - } -#endif #if defined(HAS_COMPUTECUMULATIVESUMROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2; @@ -5260,12 +5289,6 @@ int ARGBBlur(const uint8_t* src_argb, if (radius <= 0 || height <= 1) { return -1; } -#if defined(HAS_CUMULATIVESUMTOAVERAGEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ComputeCumulativeSumRow = ComputeCumulativeSumRow_SSE2; - CumulativeSumToAverageRow = CumulativeSumToAverageRow_SSE2; - } -#endif #if defined(HAS_CUMULATIVESUMTOAVERAGEROW_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ComputeCumulativeSumRow = ComputeCumulativeSumRow_AVX2; @@ -5371,9 +5394,9 @@ int ARGBShade(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } -#if defined(HAS_ARGBSHADEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(width, 4)) { - ARGBShadeRow = ARGBShadeRow_SSE2; +#if defined(HAS_ARGBSHADEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBShadeRow = ARGBShadeRow_AVX2; } #endif #if defined(HAS_ARGBSHADEROW_NEON) @@ -5539,14 +5562,6 @@ int InterpolatePlane_16(const uint16_t* src0, height = 1; src_stride0 = src_stride1 = dst_stride = 0; } -#if defined(HAS_INTERPOLATEROW_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - InterpolateRow_16 = InterpolateRow_16_Any_SSSE3; - if (IS_ALIGNED(width, 16)) { - InterpolateRow_16 = InterpolateRow_16_SSSE3; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow_16 = InterpolateRow_16_Any_AVX2; @@ -5573,14 +5588,6 @@ int InterpolatePlane_16(const uint16_t* src0, InterpolateRow_16 = InterpolateRow_16_SME; } #endif -#if defined(HAS_INTERPOLATEROW_16_LSX) - if (TestCpuFlag(kCpuHasLSX)) { - InterpolateRow_16 = InterpolateRow_16_Any_LSX; - if (IS_ALIGNED(width, 32)) { - InterpolateRow_16 = InterpolateRow_16_LSX; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD)) { InterpolateRow_16 = InterpolateRow_16_Any_WASMSIMD; @@ -5868,11 +5875,21 @@ int GaussPlane_F32(const float* src, src_stride = -src_stride; } +#if defined(HAS_GAUSSCOL_F32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + GaussCol_F32 = GaussCol_F32_AVX2; + } +#endif #if defined(HAS_GAUSSCOL_F32_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { GaussCol_F32 = GaussCol_F32_NEON; } #endif +#if defined(HAS_GAUSSROW_F32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + GaussRow_F32 = GaussRow_F32_AVX2; + } +#endif #if defined(HAS_GAUSSROW_F32_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(width, 8)) { GaussRow_F32 = GaussRow_F32_NEON; @@ -6026,9 +6043,9 @@ static int ARGBSobelize(const uint8_t* src_argb, } #endif -#if defined(HAS_SOBELYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelYRow = SobelYRow_SSE2; +#if defined(HAS_SOBELYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelYRow = SobelYRow_AVX2; } #endif #if defined(HAS_SOBELYROW_NEON) @@ -6036,9 +6053,9 @@ static int ARGBSobelize(const uint8_t* src_argb, SobelYRow = SobelYRow_NEON; } #endif -#if defined(HAS_SOBELXROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelXRow = SobelXRow_SSE2; +#if defined(HAS_SOBELXROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelXRow = SobelXRow_AVX2; } #endif #if defined(HAS_SOBELXROW_NEON) @@ -6106,11 +6123,11 @@ int ARGBSobel(const uint8_t* src_argb, int height) { void (*SobelRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width) = SobelRow_C; -#if defined(HAS_SOBELROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelRow = SobelRow_Any_SSE2; +#if defined(HAS_SOBELROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelRow = SobelRow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - SobelRow = SobelRow_SSE2; + SobelRow = SobelRow_AVX2; } } #endif @@ -6152,11 +6169,11 @@ int ARGBSobelToPlane(const uint8_t* src_argb, int height) { void (*SobelToPlaneRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_, int width) = SobelToPlaneRow_C; -#if defined(HAS_SOBELTOPLANEROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelToPlaneRow = SobelToPlaneRow_Any_SSE2; - if (IS_ALIGNED(width, 16)) { - SobelToPlaneRow = SobelToPlaneRow_SSE2; +#if defined(HAS_SOBELTOPLANEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelToPlaneRow = SobelToPlaneRow_Any_AVX2; + if (IS_ALIGNED(width, 32)) { + SobelToPlaneRow = SobelToPlaneRow_AVX2; } } #endif @@ -6199,11 +6216,11 @@ int ARGBSobelXY(const uint8_t* src_argb, int height) { void (*SobelXYRow)(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width) = SobelXYRow_C; -#if defined(HAS_SOBELXYROW_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - SobelXYRow = SobelXYRow_Any_SSE2; +#if defined(HAS_SOBELXYROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + SobelXYRow = SobelXYRow_Any_AVX2; if (IS_ALIGNED(width, 16)) { - SobelXYRow = SobelXYRow_SSE2; + SobelXYRow = SobelXYRow_AVX2; } } #endif @@ -6340,6 +6357,11 @@ int HalfFloatPlane(const uint16_t* src_y, } } #endif +#if defined(HAS_HALFFLOATROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + HalfFloatRow = HalfFloatRow_AVX512BW; + } +#endif #if defined(HAS_HALFFLOATROW_NEON) if (TestCpuFlag(kCpuHasNEON) #if defined(__arm__) @@ -6394,6 +6416,14 @@ int ByteToFloat(const uint8_t* src_y, float* dst_y, float scale, int width) { if (!src_y || !dst_y || width <= 0) { return -1; } +#if defined(HAS_BYTETOFLOATROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ByteToFloatRow = ByteToFloatRow_Any_AVX2; + if (IS_ALIGNED(width, 16)) { + ByteToFloatRow = ByteToFloatRow_AVX2; + } + } +#endif #if defined(HAS_BYTETOFLOATROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { ByteToFloatRow = ByteToFloatRow_Any_NEON; @@ -6437,9 +6467,9 @@ int ARGBLumaColorTable(const uint8_t* src_argb, height = 1; src_stride_argb = dst_stride_argb = 0; } -#if defined(HAS_ARGBLUMACOLORTABLEROW_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(width, 4)) { - ARGBLumaColorTableRow = ARGBLumaColorTableRow_SSSE3; +#if defined(HAS_ARGBLUMACOLORTABLEROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ARGBLumaColorTableRow = ARGBLumaColorTableRow_AVX2; } #endif #if defined(HAS_ARGBLUMACOLORTABLEROW_NEON) @@ -6730,6 +6760,11 @@ int YUY2ToNV12(const uint8_t* src_yuy2, } } #endif +#if defined(HAS_YUY2TOYROW_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + YUY2ToYRow = YUY2ToYRow_AVX512BW; + } +#endif #if defined(HAS_YUY2TOYROW_NEON) if (TestCpuFlag(kCpuHasNEON)) { YUY2ToYRow = YUY2ToYRow_Any_NEON; diff --git a/source/rotate.cc b/source/rotate.cc index 59bf00824..7b27333f1 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -24,6 +24,21 @@ namespace libyuv { extern "C" { #endif +static void TransposeWxH(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + int i; + for (i = 0; i < width; ++i) { + int j; + for (j = 0; j < height; ++j) { + dst[i * dst_stride + j] = src[j * src_stride + i]; + } + } +} + LIBYUV_API void TransposePlane(const uint8_t* src, int src_stride, @@ -33,8 +48,8 @@ void TransposePlane(const uint8_t* src, int height) { int i = height; #if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX) - void (*TransposeWxH)(const uint8_t* src, int src_stride, uint8_t* dst, - int dst_stride, int width, int height) = NULL; + void (*TransposeWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst, + int dst_stride, int width, int height) = NULL; #endif #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \ defined(HAS_TRANSPOSEWX16_LASX) || defined(HAS_TRANSPOSEWX16_WASMSIMD) @@ -67,19 +82,19 @@ void TransposePlane(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEWXH_SME) if (TestCpuFlag(kCpuHasSME)) { - TransposeWxH = TransposeWxH_SME; + TransposeWxH_fn = TransposeWxH_SME; } #endif #if defined(HAS_TRANSPOSEWXH_HVX) if (TestCpuFlag(kCpuHasHVX)) { - TransposeWxH = TransposeWxH_HVX; + TransposeWxH_fn = TransposeWxH_HVX; } #endif -#if defined(HAS_TRANSPOSEWX8_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - TransposeWx8 = TransposeWx8_Any_SSSE3; +#if defined(HAS_TRANSPOSEWX8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx8 = TransposeWx8_Any_AVX2; if (IS_ALIGNED(width, 16)) { - TransposeWx8 = TransposeWx8_SSSE3; + TransposeWx8 = TransposeWx8_AVX2; } } #endif @@ -130,8 +145,8 @@ void TransposePlane(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEWXH_SME) || defined(HAS_TRANSPOSEWXH_HVX) - if (TransposeWxH) { - TransposeWxH(src, src_stride, dst, dst_stride, width, height); + if (TransposeWxH_fn) { + TransposeWxH_fn(src, src_stride, dst, dst_stride, width, height); return; } #endif @@ -159,7 +174,7 @@ void TransposePlane(const uint8_t* src, #endif if (i > 0) { - TransposeWxH_C(src, src_stride, dst, dst_stride, width, i); + TransposeWxH(src, src_stride, dst, dst_stride, width, i); } } @@ -237,10 +252,7 @@ void RotatePlane180(const uint8_t* src, #endif #if defined(HAS_MIRRORROW_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { - MirrorRow = MirrorRow_Any_AVX512BW; - if (IS_ALIGNED(width, 64)) { - MirrorRow = MirrorRow_AVX512BW; - } + MirrorRow = MirrorRow_AVX512BW; } #endif #if defined(HAS_MIRRORROW_LSX) @@ -341,6 +353,24 @@ void RotatePlane180(const uint8_t* src, free_aligned_buffer_64(row); } +static void TransposeUVWxH(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width, + int height) { + int i; + for (i = 0; i < width * 2; i += 2) { + int j; + for (j = 0; j < height; ++j) { + dst_a[((i >> 1) * dst_stride_a) + j] = src[i + (j * src_stride)]; + dst_b[((i >> 1) * dst_stride_b) + j] = src[i + (j * src_stride) + 1]; + } + } +} + LIBYUV_API void SplitTransposeUV(const uint8_t* src, int src_stride, @@ -352,9 +382,9 @@ void SplitTransposeUV(const uint8_t* src, int height) { int i = height; #if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX) - void (*TransposeUVWxH)(const uint8_t* src, int src_stride, uint8_t* dst_a, - int dst_stride_a, uint8_t* dst_b, int dst_stride_b, - int width, int height) = NULL; + void (*TransposeUVWxH_fn)(const uint8_t* src, int src_stride, uint8_t* dst_a, + int dst_stride_a, uint8_t* dst_b, int dst_stride_b, + int width, int height) = NULL; #endif #if defined(HAS_TRANSPOSEUVWX16_AVX2) || \ defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \ @@ -393,19 +423,19 @@ void SplitTransposeUV(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEUVWXH_SME) if (TestCpuFlag(kCpuHasSME)) { - TransposeUVWxH = TransposeUVWxH_SME; + TransposeUVWxH_fn = TransposeUVWxH_SME; } #endif #if defined(HAS_TRANSPOSEUVWXH_HVX) if (TestCpuFlag(kCpuHasHVX)) { - TransposeUVWxH = TransposeUVWxH_HVX; + TransposeUVWxH_fn = TransposeUVWxH_HVX; } #endif -#if defined(HAS_TRANSPOSEUVWX8_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - TransposeUVWx8 = TransposeUVWx8_Any_SSE2; +#if defined(HAS_TRANSPOSEUVWX8_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeUVWx8 = TransposeUVWx8_Any_AVX2; if (IS_ALIGNED(width, 8)) { - TransposeUVWx8 = TransposeUVWx8_SSE2; + TransposeUVWx8 = TransposeUVWx8_AVX2; } } #endif @@ -440,9 +470,9 @@ void SplitTransposeUV(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEUVWXH_SME) || defined(HAS_TRANSPOSEUVWXH_HVX) - if (TransposeUVWxH) { - TransposeUVWxH(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, - width, i); + if (TransposeUVWxH_fn) { + TransposeUVWxH_fn(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, + width, i); return; } #endif @@ -472,8 +502,8 @@ void SplitTransposeUV(const uint8_t* src, } if (i > 0) { - TransposeUVWxH_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, - width, i); + TransposeUVWxH(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, + width, i); } } @@ -657,6 +687,21 @@ int RotatePlane(const uint8_t* src, return -1; } +static void TransposeWxH_16(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width, + int height) { + int i; + for (i = 0; i < width; ++i) { + int j; + for (j = 0; j < height; ++j) { + dst[i * dst_stride + j] = src[j * src_stride + i]; + } + } +} + static void TransposePlane_16(const uint16_t* src, int src_stride, uint16_t* dst, @@ -666,8 +711,8 @@ static void TransposePlane_16(const uint16_t* src, int i = height; #if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \ defined(HAS_TRANSPOSEWXH_16_HVX) - void (*TransposeWxH_16)(const uint16_t* src, int src_stride, uint16_t* dst, - int dst_stride, int width, int height) = NULL; + void (*TransposeWxH_16_fn)(const uint16_t* src, int src_stride, uint16_t* dst, + int dst_stride, int width, int height) = NULL; #endif #if defined(HAS_TRANSPOSEWX32_16_AVX512BW) void (*TransposeWx32_16)(const uint16_t* src, int src_stride, uint16_t* dst, @@ -695,17 +740,25 @@ static void TransposePlane_16(const uint16_t* src, #endif #if defined(HAS_TRANSPOSEWXH_16_SME) if (TestCpuFlag(kCpuHasSME)) { - TransposeWxH_16 = TransposeWxH_16_SME; + TransposeWxH_16_fn = TransposeWxH_16_SME; } #endif #if defined(HAS_TRANSPOSEWXH_16_RVV) if (TestCpuFlag(kCpuHasRVV)) { - TransposeWxH_16 = TransposeWxH_16_RVV; + TransposeWxH_16_fn = TransposeWxH_16_RVV; } #endif #if defined(HAS_TRANSPOSEWXH_16_HVX) if (TestCpuFlag(kCpuHasHVX)) { - TransposeWxH_16 = TransposeWxH_16_HVX; + TransposeWxH_16_fn = TransposeWxH_16_HVX; + } +#endif +#if defined(HAS_TRANSPOSEWX8_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx8_16 = TransposeWx8_16_Any_AVX2; + if (IS_ALIGNED(width, 8)) { + TransposeWx8_16 = TransposeWx8_16_AVX2; + } } #endif #if defined(HAS_TRANSPOSEWX16_16_AVX2) @@ -748,8 +801,8 @@ static void TransposePlane_16(const uint16_t* src, #if defined(HAS_TRANSPOSEWXH_16_SME) || defined(HAS_TRANSPOSEWXH_16_RVV) || \ defined(HAS_TRANSPOSEWXH_16_HVX) - if (TransposeWxH_16) { - TransposeWxH_16(src, src_stride, dst, dst_stride, width, height); + if (TransposeWxH_16_fn) { + TransposeWxH_16_fn(src, src_stride, dst, dst_stride, width, height); return; } #endif @@ -782,7 +835,7 @@ static void TransposePlane_16(const uint16_t* src, } if (i > 0) { - TransposeWxH_16_C(src, src_stride, dst, dst_stride, width, i); + TransposeWxH_16(src, src_stride, dst, dst_stride, width, i); } } diff --git a/source/rotate_any.cc b/source/rotate_any.cc index 499047cb4..bc203655a 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -34,8 +34,8 @@ TANY(TransposeWx8_Any_NEON, TransposeWx8_NEON, TransposeWx8_C, 7) #ifdef HAS_TRANSPOSEWX16_NEON TANY(TransposeWx16_Any_NEON, TransposeWx16_NEON, TransposeWx16_C, 15) #endif -#ifdef HAS_TRANSPOSEWX8_SSSE3 -TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15) +#ifdef HAS_TRANSPOSEWX8_AVX2 +TANY(TransposeWx8_Any_AVX2, TransposeWx8_AVX2, TransposeWx8_C, 15) #endif #ifdef HAS_TRANSPOSEWX16_AVX2 TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15) @@ -71,8 +71,8 @@ TANY(TransposeWx16_Any_WASMSIMD, TransposeWx16_WASMSIMD, TransposeWx16_C, 15) #ifdef HAS_TRANSPOSEUVWX8_NEON TUVANY(TransposeUVWx8_Any_NEON, TransposeUVWx8_NEON, TransposeUVWx8_C, 7) #endif -#ifdef HAS_TRANSPOSEUVWX8_SSE2 -TUVANY(TransposeUVWx8_Any_SSE2, TransposeUVWx8_SSE2, TransposeUVWx8_C, 7) +#ifdef HAS_TRANSPOSEUVWX8_AVX2 +TUVANY(TransposeUVWx8_Any_AVX2, TransposeUVWx8_AVX2, TransposeUVWx8_C, 7) #endif #ifdef HAS_TRANSPOSEUVWX16_AVX2 TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7) @@ -111,8 +111,8 @@ TUVANY(TransposeUVWx16_Any_WASMSIMD, } \ } -#ifdef HAS_TRANSPOSEWX4_32_SSE2 -TANY32(TransposeWx4_32_Any_SSE2, TransposeWx4_32_SSE2, TransposeWxH_32_C, 3) +#ifdef HAS_TRANSPOSEWX4_32_AVX2 +TANY32(TransposeWx4_32_Any_AVX2, TransposeWx4_32_AVX2, TransposeWxH_32_C, 3) #endif #ifdef HAS_TRANSPOSEWX4_32_LSX TANY32(TransposeWx4_32_Any_LSX, TransposeWx4_32_LSX, TransposeWxH_32_C, 3) @@ -155,13 +155,20 @@ TANY32_SRC(TransposeWx8_32_Any_AVX2, TransposeWx8_32_AVX2, 3, 8) TPOS_SIMD(src, src_stride, dst, dst_stride, n); \ } \ ptrdiff_t np = n; \ - TransposeWxH_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \ - r, HEIGHT); \ + TransposeWx8_16_C(src + np, src_stride, dst + np * dst_stride, dst_stride, \ + r); \ + if ((HEIGHT) == 16) { \ + TransposeWx8_16_C(src + np + 8 * src_stride, src_stride, \ + dst + np * dst_stride + 8, dst_stride, r); \ + } \ } #ifdef HAS_TRANSPOSEWX8_16_NEON TANY16(TransposeWx8_16_Any_NEON, TransposeWx8_16_NEON, 7, 8) #endif +#ifdef HAS_TRANSPOSEWX8_16_AVX2 +TANY16(TransposeWx8_16_Any_AVX2, TransposeWx8_16_AVX2, 7, 8) +#endif #ifdef HAS_TRANSPOSEWX16_16_AVX2 TANY16(TransposeWx16_16_Any_AVX2, TransposeWx16_16_AVX2, 7, 16) #endif diff --git a/source/rotate_argb.cc b/source/rotate_argb.cc index 00900d175..1b0bf7417 100644 --- a/source/rotate_argb.cc +++ b/source/rotate_argb.cc @@ -31,7 +31,8 @@ static int ARGBTranspose(const uint8_t* src_argb, int width, int height) { #if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \ - defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) + defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \ + defined(HAS_TRANSPOSEWXH_32_AVX2) void (*TransposeWxH_32)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width, int height) = NULL; #endif @@ -49,11 +50,11 @@ static int ARGBTranspose(const uint8_t* src_argb, if (src_stride_argb & 3) { return -1; } -#if defined(HAS_TRANSPOSEWX4_32_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - TransposeWx4_32 = TransposeWx4_32_Any_SSE2; +#if defined(HAS_TRANSPOSEWX4_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx4_32 = TransposeWx4_32_Any_AVX2; if (IS_ALIGNED(height, 4)) { // Width of dest. - TransposeWx4_32 = TransposeWx4_32_SSE2; + TransposeWx4_32 = TransposeWx4_32_AVX2; } } #endif @@ -65,6 +66,11 @@ static int ARGBTranspose(const uint8_t* src_argb, } } #endif +#if defined(HAS_TRANSPOSEWXH_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWxH_32 = TransposeWxH_32_AVX2; + } +#endif #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) if (TestCpuFlag(kCpuHasAVX512BW)) { TransposeWx16_32 = TransposeWx16_32_AVX512BW; @@ -123,14 +129,6 @@ static int ARGBTranspose(const uint8_t* src_argb, } #endif -#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \ - defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) - if (TransposeWxH_32) { - TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width, - height); - return 0; - } -#endif #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) if (TransposeWx16_32) { while (height >= 16) { @@ -142,6 +140,17 @@ static int ARGBTranspose(const uint8_t* src_argb, } } #endif +#if defined(HAS_TRANSPOSEWXH_32_NEON) || defined(HAS_TRANSPOSEWXH_32_SME) || \ + defined(HAS_TRANSPOSEWXH_32_RVV) || defined(HAS_TRANSPOSEWXH_32_HVX) || \ + defined(HAS_TRANSPOSEWXH_32_AVX2) + if (TransposeWxH_32) { + if (height > 0) { + TransposeWxH_32(src_argb, src_stride_argb, dst_argb, dst_stride_argb, + width, height); + } + return 0; + } +#endif #if defined(HAS_TRANSPOSEWX8_32_AVX2) if (TransposeWx8_32) { while (height >= 8) { diff --git a/source/rotate_common.cc b/source/rotate_common.cc index 0abab4f7e..e4aab102f 100644 --- a/source/rotate_common.cc +++ b/source/rotate_common.cc @@ -90,39 +90,6 @@ void TransposeUVWx16_C(const uint8_t* src, dst_stride_a, (dst_b + 8), dst_stride_b, width); } -void TransposeWxH_C(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width, - int height) { - int i; - for (i = 0; i < width; ++i) { - int j; - for (j = 0; j < height; ++j) { - dst[i * dst_stride + j] = src[j * src_stride + i]; - } - } -} - -void TransposeUVWxH_C(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width, - int height) { - int i; - for (i = 0; i < width * 2; i += 2) { - int j; - for (j = 0; j < height; ++j) { - dst_a[((i >> 1) * dst_stride_a) + j] = src[i + (j * src_stride)]; - dst_b[((i >> 1) * dst_stride_b) + j] = src[i + (j * src_stride) + 1]; - } - } -} - void TransposeWx8_16_C(const uint16_t* src, int src_stride, uint16_t* dst, @@ -143,21 +110,6 @@ void TransposeWx8_16_C(const uint16_t* src, } } -void TransposeWxH_16_C(const uint16_t* src, - int src_stride, - uint16_t* dst, - int dst_stride, - int width, - int height) { - int i; - for (i = 0; i < width; ++i) { - int j; - for (j = 0; j < height; ++j) { - dst[i * dst_stride + j] = src[j * src_stride + i]; - } - } -} - static inline void Copy4(uint8_t* dst, const uint8_t* src) { dst[0] = src[0]; dst[1] = src[1]; diff --git a/source/rotate_gcc.cc b/source/rotate_gcc.cc index d9456a842..251054f2a 100644 --- a/source/rotate_gcc.cc +++ b/source/rotate_gcc.cc @@ -11,6 +11,8 @@ #include "libyuv/rotate_row.h" #include "libyuv/row.h" +#include + #ifdef __cplusplus namespace libyuv { extern "C" { @@ -23,152 +25,135 @@ extern "C" { // asm statement, and other OS ABIs (e.g. Windows) differ in which registers // are callee-saved. List "memory" first, then "cc", then registers. -// This module is for GCC and clang x64. 32 bit x86 uses rotate_win.cc. -#if !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && \ - !defined(LIBYUV_ENABLE_ROWWIN) +// This module is for GCC and clang x86 and x64. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__)) && \ + !defined(LIBYUV_ENABLE_ROWWIN) && \ + (!defined(_MSC_VER) || defined(__clang__)) -#if defined(HAS_TRANSPOSEWX8_SSSE3) -// Transpose 16x8. 64 bit. -void TransposeWx8_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { +#if defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2) +// Transpose 16x8 bytes using 2 lanes of AVX2. Source row r is in lane r / 4 +// of ymm(r % 4). Byte and word unpacks produce 4-row column chunks in each +// lane; vperm2i128 + dword unpack join rows 0..3 and 4..7. Each 128-bit lane +// holds 8 bytes for dst_a (low 64 bits) and 8 bytes for dst_b (high 64 bits). +// Width is a multiple of 16 bytes. +static void TransposeWx8_Byte_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int byte_width) { + const ptrdiff_t d_a = dst_stride_a; + const ptrdiff_t d_b = dst_stride_b; + uintptr_t temp; asm volatile( - // Read in the data from the source pointer. - // First round of bit swap. - LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%0,%3),%%xmm1 \n" - "lea (%0,%3,2),%0 \n" - "movdqa %%xmm0,%%xmm8 \n" - "punpcklbw %%xmm1,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm8 \n" - "movdqu (%0),%%xmm2 \n" - "movdqa %%xmm0,%%xmm1 \n" - "movdqa %%xmm8,%%xmm9 \n" - "palignr $0x8,%%xmm1,%%xmm1 \n" - "palignr $0x8,%%xmm9,%%xmm9 \n" - "movdqu (%0,%3),%%xmm3 \n" - "lea (%0,%3,2),%0 \n" - "movdqa %%xmm2,%%xmm10 \n" - "punpcklbw %%xmm3,%%xmm2 \n" - "punpckhbw %%xmm3,%%xmm10 \n" - "movdqa %%xmm2,%%xmm3 \n" - "movdqa %%xmm10,%%xmm11 \n" - "movdqu (%0),%%xmm4 \n" - "palignr $0x8,%%xmm3,%%xmm3 \n" - "palignr $0x8,%%xmm11,%%xmm11 \n" - "movdqu (%0,%3),%%xmm5 \n" - "lea (%0,%3,2),%0 \n" - "movdqa %%xmm4,%%xmm12 \n" - "punpcklbw %%xmm5,%%xmm4 \n" - "punpckhbw %%xmm5,%%xmm12 \n" - "movdqa %%xmm4,%%xmm5 \n" - "movdqa %%xmm12,%%xmm13 \n" - "movdqu (%0),%%xmm6 \n" - "palignr $0x8,%%xmm5,%%xmm5 \n" - "palignr $0x8,%%xmm13,%%xmm13 \n" - "movdqu (%0,%3),%%xmm7 \n" - "lea (%0,%3,2),%0 \n" - "movdqa %%xmm6,%%xmm14 \n" - "punpcklbw %%xmm7,%%xmm6 \n" - "punpckhbw %%xmm7,%%xmm14 \n" - "neg %3 \n" - "movdqa %%xmm6,%%xmm7 \n" - "movdqa %%xmm14,%%xmm15 \n" - "lea 0x10(%0,%3,8),%0 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "palignr $0x8,%%xmm15,%%xmm15 \n" - "neg %3 \n" - // Second round of bit swap. - "punpcklwd %%xmm2,%%xmm0 \n" - "punpcklwd %%xmm3,%%xmm1 \n" - "movdqa %%xmm0,%%xmm2 \n" - "movdqa %%xmm1,%%xmm3 \n" - "palignr $0x8,%%xmm2,%%xmm2 \n" - "palignr $0x8,%%xmm3,%%xmm3 \n" - "punpcklwd %%xmm6,%%xmm4 \n" - "punpcklwd %%xmm7,%%xmm5 \n" - "movdqa %%xmm4,%%xmm6 \n" - "movdqa %%xmm5,%%xmm7 \n" - "palignr $0x8,%%xmm6,%%xmm6 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "punpcklwd %%xmm10,%%xmm8 \n" - "punpcklwd %%xmm11,%%xmm9 \n" - "movdqa %%xmm8,%%xmm10 \n" - "movdqa %%xmm9,%%xmm11 \n" - "palignr $0x8,%%xmm10,%%xmm10 \n" - "palignr $0x8,%%xmm11,%%xmm11 \n" - "punpcklwd %%xmm14,%%xmm12 \n" - "punpcklwd %%xmm15,%%xmm13 \n" - "movdqa %%xmm12,%%xmm14 \n" - "movdqa %%xmm13,%%xmm15 \n" - "palignr $0x8,%%xmm14,%%xmm14 \n" - "palignr $0x8,%%xmm15,%%xmm15 \n" - // Third round of bit swap. - // Write to the destination pointer. - "punpckldq %%xmm4,%%xmm0 \n" - "movq %%xmm0,(%1) \n" - "movdqa %%xmm0,%%xmm4 \n" - "palignr $0x8,%%xmm4,%%xmm4 \n" - "movq %%xmm4,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm6,%%xmm2 \n" - "movdqa %%xmm2,%%xmm6 \n" - "movq %%xmm2,(%1) \n" - "palignr $0x8,%%xmm6,%%xmm6 \n" - "punpckldq %%xmm5,%%xmm1 \n" - "movq %%xmm6,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "movdqa %%xmm1,%%xmm5 \n" - "movq %%xmm1,(%1) \n" - "palignr $0x8,%%xmm5,%%xmm5 \n" - "movq %%xmm5,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm7,%%xmm3 \n" - "movq %%xmm3,(%1) \n" - "movdqa %%xmm3,%%xmm7 \n" - "palignr $0x8,%%xmm7,%%xmm7 \n" - "movq %%xmm7,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm12,%%xmm8 \n" - "movq %%xmm8,(%1) \n" - "movdqa %%xmm8,%%xmm12 \n" - "palignr $0x8,%%xmm12,%%xmm12 \n" - "movq %%xmm12,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm14,%%xmm10 \n" - "movdqa %%xmm10,%%xmm14 \n" - "movq %%xmm10,(%1) \n" - "palignr $0x8,%%xmm14,%%xmm14 \n" - "punpckldq %%xmm13,%%xmm9 \n" - "movq %%xmm14,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "movdqa %%xmm9,%%xmm13 \n" - "movq %%xmm9,(%1) \n" - "palignr $0x8,%%xmm13,%%xmm13 \n" - "movq %%xmm13,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" - "punpckldq %%xmm15,%%xmm11 \n" - "movq %%xmm11,(%1) \n" - "movdqa %%xmm11,%%xmm15 \n" - "palignr $0x8,%%xmm15,%%xmm15 \n" - "sub $0x10,%2 \n" - "movq %%xmm15,(%1,%4) \n" - "lea (%1,%4,2),%1 \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[src_stride]),%%xmm1 \n" + "lea (%[src],%[src_stride],2),%[temp] \n" + "lea 0x10(%[src]),%[src] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[src_stride]),%%xmm3 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[src_stride]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[src_stride],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[src_stride]),%%ymm3,%%ymm3 \n" + + "vpunpcklbw %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckhbw %%ymm1,%%ymm0,%%ymm5 \n" + "vpunpcklbw %%ymm3,%%ymm2,%%ymm6 \n" + "vpunpckhbw %%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpcklwd %%ymm6,%%ymm4,%%ymm0 \n" + "vpunpckhwd %%ymm6,%%ymm4,%%ymm1 \n" + "vpunpcklwd %%ymm7,%%ymm5,%%ymm2 \n" + "vpunpckhwd %%ymm7,%%ymm5,%%ymm3 \n" + + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm4 \n" + "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm5 \n" + "vperm2i128 $0x20,%%ymm3,%%ymm2,%%ymm6 \n" + "vperm2i128 $0x31,%%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpckldq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpckldq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhdq %%ymm7,%%ymm6,%%ymm3 \n" + + "vmovq %%xmm0,(%[dst_a]) \n" + "vmovhpd %%xmm0,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm1,(%[dst_a]) \n" + "vmovhpd %%xmm1,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vextracti128 $1,%%ymm0,%%xmm4 \n" + "vextracti128 $1,%%ymm1,%%xmm5 \n" + "vmovq %%xmm4,(%[dst_a]) \n" + "vmovhpd %%xmm4,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm5,(%[dst_a]) \n" + "vmovhpd %%xmm5,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm2,(%[dst_a]) \n" + "vmovhpd %%xmm2,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm3,(%[dst_a]) \n" + "vmovhpd %%xmm3,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vextracti128 $1,%%ymm2,%%xmm6 \n" + "vextracti128 $1,%%ymm3,%%xmm7 \n" + "vmovq %%xmm6,(%[dst_a]) \n" + "vmovhpd %%xmm6,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "vmovq %%xmm7,(%[dst_a]) \n" + "vmovhpd %%xmm7,(%[dst_b]) \n" + "add %[dst_stride_a],%[dst_a] \n" + "add %[dst_stride_b],%[dst_b] \n" + "subl $0x10,%[width] \n" "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "r"((ptrdiff_t)(src_stride)), // %3 - "r"((ptrdiff_t)(dst_stride)) // %4 + "vzeroupper \n" + : [src] "+r"(src), [dst_a] "+r"(dst_a), [dst_b] "+r"(dst_b), + [width] "+rm"(byte_width), [temp] "=&r"(temp) + : [src_stride] "r"((ptrdiff_t)src_stride), + [dst_stride_a] "rm"(d_a), + [dst_stride_b] "rm"(d_b) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", - "xmm15"); + "xmm7"); } -#endif // defined(HAS_TRANSPOSEWX8_SSSE3) +#endif // defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2) + +#if defined(HAS_TRANSPOSEWX8_AVX2) +void TransposeWx8_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx8_Byte_AVX2(src, src_stride, dst, dst_stride * 2, dst + dst_stride, + dst_stride * 2, width); +} +#endif // defined(HAS_TRANSPOSEWX8_AVX2) + +#if defined(HAS_TRANSPOSEUVWX8_AVX2) +void TransposeUVWx8_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeWx8_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, + dst_stride_b, width * 2); +} +#endif // defined(HAS_TRANSPOSEUVWX8_AVX2) #if defined(HAS_TRANSPOSEWX16_AVX512BW) || defined(HAS_TRANSPOSEUVWX16_AVX512BW) // Dword permutes to gather 16 rows of a column from 4 lanes. @@ -411,6 +396,7 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(__x86_64__) #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2) // Transpose 16x16 bytes using 2 lanes of AVX2. Source row r is in lane r / 8 // of ymm(r % 8). 3 unpack stages leave each lane with 2 columns x 8 rows, and @@ -549,122 +535,37 @@ void TransposeUVWx16_AVX2(const uint8_t* src, dst_stride_b, width * 2); } #endif // defined(HAS_TRANSPOSEUVWX16_AVX2) - -// Transpose UV 8x8. 64 bit. -#if defined(HAS_TRANSPOSEUVWX8_SSE2) -void TransposeUVWx8_SSE2(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width) { - asm volatile( - // Read in the data from the source pointer. - // First round of bit swap. - LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%0,%4),%%xmm1 \n" - "lea (%0,%4,2),%0 \n" - "movdqa %%xmm0,%%xmm8 \n" - "punpcklbw %%xmm1,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm8 \n" - "movdqa %%xmm8,%%xmm1 \n" - "movdqu (%0),%%xmm2 \n" - "movdqu (%0,%4),%%xmm3 \n" - "lea (%0,%4,2),%0 \n" - "movdqa %%xmm2,%%xmm8 \n" - "punpcklbw %%xmm3,%%xmm2 \n" - "punpckhbw %%xmm3,%%xmm8 \n" - "movdqa %%xmm8,%%xmm3 \n" - "movdqu (%0),%%xmm4 \n" - "movdqu (%0,%4),%%xmm5 \n" - "lea (%0,%4,2),%0 \n" - "movdqa %%xmm4,%%xmm8 \n" - "punpcklbw %%xmm5,%%xmm4 \n" - "punpckhbw %%xmm5,%%xmm8 \n" - "movdqa %%xmm8,%%xmm5 \n" - "movdqu (%0),%%xmm6 \n" - "movdqu (%0,%4),%%xmm7 \n" - "lea (%0,%4,2),%0 \n" - "movdqa %%xmm6,%%xmm8 \n" - "punpcklbw %%xmm7,%%xmm6 \n" - "neg %4 \n" - "lea 0x10(%0,%4,8),%0 \n" - "punpckhbw %%xmm7,%%xmm8 \n" - "movdqa %%xmm8,%%xmm7 \n" - "neg %4 \n" - // Second round of bit swap. - "movdqa %%xmm0,%%xmm8 \n" - "movdqa %%xmm1,%%xmm9 \n" - "punpckhwd %%xmm2,%%xmm8 \n" - "punpckhwd %%xmm3,%%xmm9 \n" - "punpcklwd %%xmm2,%%xmm0 \n" - "punpcklwd %%xmm3,%%xmm1 \n" - "movdqa %%xmm8,%%xmm2 \n" - "movdqa %%xmm9,%%xmm3 \n" - "movdqa %%xmm4,%%xmm8 \n" - "movdqa %%xmm5,%%xmm9 \n" - "punpckhwd %%xmm6,%%xmm8 \n" - "punpckhwd %%xmm7,%%xmm9 \n" - "punpcklwd %%xmm6,%%xmm4 \n" - "punpcklwd %%xmm7,%%xmm5 \n" - "movdqa %%xmm8,%%xmm6 \n" - "movdqa %%xmm9,%%xmm7 \n" - // Third round of bit swap. - // Write to the destination pointer. - "movdqa %%xmm0,%%xmm8 \n" - "punpckldq %%xmm4,%%xmm0 \n" - "movlpd %%xmm0,(%1) \n" // Write back U channel - "movhpd %%xmm0,(%2) \n" // Write back V channel - "punpckhdq %%xmm4,%%xmm8 \n" - "movlpd %%xmm8,(%1,%5) \n" - "lea (%1,%5,2),%1 \n" - "movhpd %%xmm8,(%2,%6) \n" - "lea (%2,%6,2),%2 \n" - "movdqa %%xmm2,%%xmm8 \n" - "punpckldq %%xmm6,%%xmm2 \n" - "movlpd %%xmm2,(%1) \n" - "movhpd %%xmm2,(%2) \n" - "punpckhdq %%xmm6,%%xmm8 \n" - "movlpd %%xmm8,(%1,%5) \n" - "lea (%1,%5,2),%1 \n" - "movhpd %%xmm8,(%2,%6) \n" - "lea (%2,%6,2),%2 \n" - "movdqa %%xmm1,%%xmm8 \n" - "punpckldq %%xmm5,%%xmm1 \n" - "movlpd %%xmm1,(%1) \n" - "movhpd %%xmm1,(%2) \n" - "punpckhdq %%xmm5,%%xmm8 \n" - "movlpd %%xmm8,(%1,%5) \n" - "lea (%1,%5,2),%1 \n" - "movhpd %%xmm8,(%2,%6) \n" - "lea (%2,%6,2),%2 \n" - "movdqa %%xmm3,%%xmm8 \n" - "punpckldq %%xmm7,%%xmm3 \n" - "movlpd %%xmm3,(%1) \n" - "movhpd %%xmm3,(%2) \n" - "punpckhdq %%xmm7,%%xmm8 \n" - "sub $0x8,%3 \n" - "movlpd %%xmm8,(%1,%5) \n" - "lea (%1,%5,2),%1 \n" - "movhpd %%xmm8,(%2,%6) \n" - "lea (%2,%6,2),%2 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst_a), // %1 - "+r"(dst_b), // %2 - "+r"(width) // %3 - : "r"((ptrdiff_t)(src_stride)), // %4 - "r"((ptrdiff_t)(dst_stride_a)), // %5 - "r"((ptrdiff_t)(dst_stride_b)) // %6 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7", "xmm8", "xmm9"); +#else // !defined(__x86_64__) +#if defined(HAS_TRANSPOSEWX16_AVX2) +void TransposeWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx8_AVX2(src, src_stride, dst, dst_stride, width); + TransposeWx8_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst + 8, + dst_stride, width); } -#endif // defined(HAS_TRANSPOSEUVWX8_SSE2) +#endif // defined(HAS_TRANSPOSEWX16_AVX2) + +#if defined(HAS_TRANSPOSEUVWX16_AVX2) +void TransposeUVWx16_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + TransposeUVWx8_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, + width); + TransposeUVWx8_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst_a + 8, + dst_stride_a, dst_b + 8, dst_stride_b, width); +} +#endif // defined(HAS_TRANSPOSEUVWX16_AVX2) +#endif // defined(__x86_64__) #if defined(HAS_TRANSPOSEWX16_16_AVX2) +#if defined(__x86_64__) void TransposeWx16_16_AVX2(const uint16_t* src, int src_stride, uint16_t* dst, @@ -758,8 +659,92 @@ void TransposeWx16_16_AVX2(const uint16_t* src, "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); } +#endif // defined(__x86_64__) #endif // defined(HAS_TRANSPOSEWX16_16_AVX2) +#if defined(HAS_TRANSPOSEWX8_16_AVX2) +void TransposeWx8_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = (ptrdiff_t)src_stride * 2; + const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; + const uint8_t* src_b = (const uint8_t*)src; + uint8_t* dst_b = (uint8_t*)dst; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n" + "add $0x10,%[src] \n" + + "vpunpcklwd %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckhwd %%ymm1,%%ymm0,%%ymm5 \n" + "vpunpcklwd %%ymm3,%%ymm2,%%ymm6 \n" + "vpunpckhwd %%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpckldq %%ymm6,%%ymm4,%%ymm0 \n" + "vpunpckhdq %%ymm6,%%ymm4,%%ymm1 \n" + "vpunpckldq %%ymm7,%%ymm5,%%ymm2 \n" + "vpunpckhdq %%ymm7,%%ymm5,%%ymm3 \n" + + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpermq $0xd8,%%ymm3,%%ymm3 \n" + + "vmovdqu %%xmm0,(%[dst]) \n" + "vextracti128 $1,%%ymm0,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%xmm1,(%[dst]) \n" + "vextracti128 $1,%%ymm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%xmm2,(%[dst]) \n" + "vextracti128 $1,%%ymm2,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%xmm3,(%[dst]) \n" + "vextracti128 $1,%%ymm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "subl $0x8,%[w] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_b), + [dst] "+r"(dst_b), +#if defined(__i386__) + [w] "+m"(width), +#else + [w] "+rm"(width), +#endif + [temp] "=&r"(temp) + : [s] "r"(s), + [d] "r"(d) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // defined(HAS_TRANSPOSEWX8_16_AVX2) + +#if defined(HAS_TRANSPOSEWX16_16_AVX2) && !defined(__x86_64__) +void TransposeWx16_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + TransposeWx8_16_AVX2(src, src_stride, dst, dst_stride, width); + TransposeWx8_16_AVX2(src + 8 * (ptrdiff_t)src_stride, src_stride, dst + 8, + dst_stride, width); +} +#endif // defined(HAS_TRANSPOSEWX16_16_AVX2) && !defined(__x86_64__) + #if defined(HAS_TRANSPOSEWX32_16_AVX512BW) // Transpose 32 rows of 16 bit plane (arbitrary width) using 8x32 tiles in // zmm0..zmm15 with k1-masked right tail. @@ -1033,78 +1018,93 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src, } #endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW) -#if defined(HAS_TRANSPOSEWX4_32_SSE2) -// 4 values, little endian view -// a b c d -// e f g h -// i j k l -// m n o p - -// transpose 2x2 -// a e b f from row 0, 1 -// i m j n from row 2, 3 -// c g d h from row 0, 1 -// k o l p from row 2, 3 - -// transpose 4x4 -// a e i m from row 0, 1 -// b f j n from row 0, 1 -// c g k o from row 2, 3 -// d h l p from row 2, 3 - -// Transpose 32 bit values (ARGB) -void TransposeWx4_32_SSE2(const uint8_t* src, +#if defined(HAS_TRANSPOSEWX4_32_AVX2) +// Transpose 32 bit values (ARGB). Read a column of 4 pixels across 8 rows per +// AVX2 iteration (with a 4-row VEX 128-bit tail) and write 4 destination rows. +void TransposeWx4_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + uintptr_t temp; asm volatile( - // Main loop transpose 4x4. Read a column, write a row. - "1: \n" - "movdqu (%0),%%xmm0 \n" // a b c d - "movdqu (%0,%3),%%xmm1 \n" // e f g h - "lea (%0,%3,2),%0 \n" // src += stride * 2 - "movdqu (%0),%%xmm2 \n" // i j k l - "movdqu (%0,%3),%%xmm3 \n" // m n o p - "lea (%0,%3,2),%0 \n" // src += stride * 2 + "subl $0x8,%[width] \n" + "jl 2f \n" - // Transpose 2x2 - "movdqa %%xmm0,%%xmm4 \n" - "movdqa %%xmm2,%%xmm5 \n" - "movdqa %%xmm0,%%xmm6 \n" - "movdqa %%xmm2,%%xmm7 \n" - "punpckldq %%xmm1,%%xmm4 \n" // a e b f from row 0, 1 - "punpckldq %%xmm3,%%xmm5 \n" // i m j n from row 2, 3 - "punpckhdq %%xmm1,%%xmm6 \n" // c g d h from row 0, 1 - "punpckhdq %%xmm3,%%xmm7 \n" // k o l p from row 2, 3 + // Main loop: 8 source rows of 4 pixels -> 4 dst rows of 8 pixels (32 B). + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm0,%%ymm0 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vinserti128 $1,(%[temp]),%%ymm2,%%ymm2 \n" + "vinserti128 $1,(%[temp],%[s]),%%ymm3,%%ymm3 \n" + "lea (%[temp],%[s],2),%[src] \n" - // Transpose 4x4 - "movdqa %%xmm4,%%xmm0 \n" - "movdqa %%xmm4,%%xmm1 \n" - "movdqa %%xmm6,%%xmm2 \n" - "movdqa %%xmm6,%%xmm3 \n" - "punpcklqdq %%xmm5,%%xmm0 \n" // a e i m from row 0, 1 - "punpckhqdq %%xmm5,%%xmm1 \n" // b f j n from row 0, 1 - "punpcklqdq %%xmm7,%%xmm2 \n" // c g k o from row 2, 3 - "punpckhqdq %%xmm7,%%xmm3 \n" // d h l p from row 2, 3 + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" - "movdqu %%xmm0,(%1) \n" - "lea 16(%1,%4),%1 \n" // dst += stride + 16 - "movdqu %%xmm1,-16(%1) \n" - "movdqu %%xmm2,-16(%1,%4) \n" - "movdqu %%xmm3,-16(%1,%4,2) \n" - "sub %4,%1 \n" - "sub $0x4,%2 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+rm"(width) // %2 - : "r"((ptrdiff_t)(src_stride)), // %3 - "r"((ptrdiff_t)(dst_stride)) // %4 + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" + + "vmovdqu %%ymm0,(%[dst]) \n" + "vmovdqu %%ymm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[temp] \n" + "vmovdqu %%ymm2,(%[temp]) \n" + "vmovdqu %%ymm3,(%[temp],%[d]) \n" + "add $0x20,%[dst] \n" + "subl $0x8,%[width] \n" + "jge 1b \n" + + // Tail: 4 source rows of 4 pixels -> 4 dst rows of 4 pixels (16 B). + "2: \n" + "addl $0x8,%[width] \n" + "je 99f \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" + + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" + + "vmovdqu %%xmm0,(%[dst]) \n" + "vmovdqu %%xmm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[temp] \n" + "vmovdqu %%xmm2,(%[temp]) \n" + "vmovdqu %%xmm3,(%[temp],%[d]) \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [width] "+rm"(width), // %[width] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } -#endif // defined(HAS_TRANSPOSEWX4_32_SSE2) +#endif // defined(HAS_TRANSPOSEWX4_32_AVX2) #if defined(HAS_TRANSPOSEWX8_32_AVX2) // Transpose 8 rows of 32 bit plane (width multiple of 4) using 4x8 YMM vertical @@ -1152,20 +1152,404 @@ void TransposeWx8_32_AVX2(const uint8_t* src, "vmovdqu %%ymm2,(%[dst]) \n" "vmovdqu %%ymm3,(%[dst],%[d]) \n" "lea (%[dst],%[d],2),%[dst] \n" - "sub $0x4,%[width] \n" + "subl $0x4,%[width] \n" "jg 1b \n" "vzeroupper \n" - : [src] "+r"(src), // %[src] - [dst] "+r"(dst), // %[dst] - [width] "+r"(width), // %[width] - [temp] "=&r"(temp) // %[temp] - : [s] "r"(s), // %[s] - [d] "r"(d) // %[d] + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [width] "+rm"(width), // %[width] + [temp] "=&r"(temp) // %[temp] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } #endif // defined(HAS_TRANSPOSEWX8_32_AVX2) +#if defined(HAS_TRANSPOSEWXH_32_AVX2) +// Transpose rem_w (1..3) columns x 8 rows of 32-bit pixels using AVX2. +static void TransposeRemWx8_32_AVX2(const uint8_t* src, + ptrdiff_t s, + uint8_t* dst, + ptrdiff_t d, + int rem_w) { + const uint8_t* temp; + asm volatile( + "cmpl $2,%[rem_w] \n" + "jl 1f \n" + "je 2f \n" + + // rem_w == 3: load 12 bytes per row from 8 rows. + "vmovq (%[src]),%%xmm0 \n" + "vpinsrd $2,8(%[src]),%%xmm0,%%xmm0 \n" + "vmovq (%[src],%[s]),%%xmm1 \n" + "vpinsrd $2,8(%[src],%[s]),%%xmm1,%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm2 \n" + "vpinsrd $2,8(%[temp]),%%xmm2,%%xmm2 \n" + "vmovq (%[temp],%[s]),%%xmm3 \n" + "vpinsrd $2,8(%[temp],%[s]),%%xmm3,%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm4 \n" + "vpinsrd $2,8(%[temp]),%%xmm4,%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n" + "vmovq (%[temp],%[s]),%%xmm5 \n" + "vpinsrd $2,8(%[temp],%[s]),%%xmm5,%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm4 \n" + "vpinsrd $2,8(%[temp]),%%xmm4,%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n" + "vmovq (%[temp],%[s]),%%xmm5 \n" + "vpinsrd $2,8(%[temp],%[s]),%%xmm5,%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n" + "jmp 3f \n" + + // rem_w == 2: load 8 bytes per row from 8 rows. + "2: \n" + "vmovq (%[src]),%%xmm0 \n" + "vmovq (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm2 \n" + "vmovq (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n" + "vmovq (%[temp],%[s]),%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n" + "vmovq (%[temp],%[s]),%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n" + "jmp 3f \n" + + // rem_w == 1: load 4 bytes per row from 8 rows. + "1: \n" + "vmovd (%[src]),%%xmm0 \n" + "vmovd (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovd (%[temp]),%%xmm2 \n" + "vmovd (%[temp],%[s]),%%xmm3 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovd (%[temp]),%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm0,%%ymm0 \n" + "vmovd (%[temp],%[s]),%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm1,%%ymm1 \n" + "lea (%[temp],%[s],2),%[temp] \n" + "vmovd (%[temp]),%%xmm4 \n" + "vinserti128 $1,%%xmm4,%%ymm2,%%ymm2 \n" + "vmovd (%[temp],%[s]),%%xmm5 \n" + "vinserti128 $1,%%xmm5,%%ymm3,%%ymm3 \n" + + "3: \n" + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm6 \n" + "vmovdqu %%ymm6,(%[dst]) \n" + "cmpl $1,%[rem_w] \n" + "jle 4f \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm7 \n" + "vmovdqu %%ymm7,(%[dst],%[d]) \n" + "cmpl $2,%[rem_w] \n" + "jle 4f \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm6 \n" + "vmovdqu %%ymm6,(%[dst],%[d],2) \n" + "4: \n" + "vzeroupper \n" + : [temp] "=&r"(temp) + : [src] "r"(src), + [s] "r"(s), + [dst] "r"(dst), + [d] "r"(d), + [rem_w] "rm"(rem_w) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} + +// Transpose rem_w (1..3) columns x 4 rows of 32-bit pixels using VEX 128-bit. +static void TransposeRemWx4_32_AVX2(const uint8_t* src, + ptrdiff_t s, + uint8_t* dst, + ptrdiff_t d, + int rem_w) { + const uint8_t* temp; + asm volatile( + "cmpl $2,%[rem_w] \n" + "jl 1f \n" + "je 2f \n" + + // rem_w == 3: load 12 bytes per row from 4 rows. + "vmovq (%[src]),%%xmm0 \n" + "vpinsrd $2,8(%[src]),%%xmm0,%%xmm0 \n" + "vmovq (%[src],%[s]),%%xmm1 \n" + "vpinsrd $2,8(%[src],%[s]),%%xmm1,%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm2 \n" + "vpinsrd $2,8(%[temp]),%%xmm2,%%xmm2 \n" + "vmovq (%[temp],%[s]),%%xmm3 \n" + "vpinsrd $2,8(%[temp],%[s]),%%xmm3,%%xmm3 \n" + "jmp 3f \n" + + // rem_w == 2: load 8 bytes per row from 4 rows. + "2: \n" + "vmovq (%[src]),%%xmm0 \n" + "vmovq (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovq (%[temp]),%%xmm2 \n" + "vmovq (%[temp],%[s]),%%xmm3 \n" + "jmp 3f \n" + + // rem_w == 1: load 4 bytes per row from 4 rows. + "1: \n" + "vmovd (%[src]),%%xmm0 \n" + "vmovd (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovd (%[temp]),%%xmm2 \n" + "vmovd (%[temp],%[s]),%%xmm3 \n" + + "3: \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm6 \n" + "vmovdqu %%xmm6,(%[dst]) \n" + "cmpl $1,%[rem_w] \n" + "jle 4f \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm7 \n" + "vmovdqu %%xmm7,(%[dst],%[d]) \n" + "cmpl $2,%[rem_w] \n" + "jle 4f \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm6 \n" + "vmovdqu %%xmm6,(%[dst],%[d],2) \n" + "4: \n" + : [temp] "=&r"(temp) + : [src] "r"(src), + [s] "r"(s), + [dst] "r"(dst), + [d] "r"(d), + [rem_w] "rm"(rem_w) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} + +// Transpose arbitrary WxH 32-bit plane using AVX2 (4x8, 8x4, 4x4, and 4x{1..3} +// tiles plus exact remainder handling). +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const int w4 = width & ~3; + const int rem_w = width & 3; + + // Tier 1: 8-row strips using 4x8 YMM tiles + rem_w x 8 YMM tail. + while (height >= 8) { + if (w4 > 0) { + TransposeWx8_32_AVX2(src, src_stride, dst, dst_stride, w4); + } + if (rem_w > 0) { + TransposeRemWx8_32_AVX2(src + (ptrdiff_t)w4 * 4, s, + dst + (ptrdiff_t)w4 * d, d, rem_w); + } + src += 8 * s; + dst += 32; + height -= 8; + } + + // Tier 2: 4-row strip using 8x4 YMM tiles + 4x4 XMM block + rem_w x 4 tail. + if (height >= 4) { + const int w8 = width & ~7; + if (w8 > 0) { + const uint8_t* src_p = src; + uint8_t* dst_p = dst; + int cnt = w8; + const uint8_t* temp; + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vmovdqu (%[src],%[s]),%%ymm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%ymm2 \n" + "vmovdqu (%[temp],%[s]),%%ymm3 \n" + "add $0x20,%[src] \n" + + "vpunpckldq %%ymm1,%%ymm0,%%ymm4 \n" + "vpunpckldq %%ymm3,%%ymm2,%%ymm5 \n" + "vpunpckhdq %%ymm1,%%ymm0,%%ymm6 \n" + "vpunpckhdq %%ymm3,%%ymm2,%%ymm7 \n" + + "vpunpcklqdq %%ymm5,%%ymm4,%%ymm0 \n" + "vpunpckhqdq %%ymm5,%%ymm4,%%ymm1 \n" + "vpunpcklqdq %%ymm7,%%ymm6,%%ymm2 \n" + "vpunpckhqdq %%ymm7,%%ymm6,%%ymm3 \n" + + "vmovdqu %%xmm0,(%[dst]) \n" + "vmovdqu %%xmm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovdqu %%xmm2,(%[dst]) \n" + "vmovdqu %%xmm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vextracti128 $1,%%ymm0,(%[dst]) \n" + "vextracti128 $1,%%ymm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vextracti128 $1,%%ymm2,(%[dst]) \n" + "vextracti128 $1,%%ymm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "subl $0x8,%[cnt] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_p), + [dst] "+r"(dst_p), + [cnt] "+rm"(cnt), + [temp] "=&r"(temp) + : [s] "r"(s), + [d] "r"(d) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + } + if (width & 4) { + const uint8_t* src_p = src + (ptrdiff_t)w8 * 4; + uint8_t* dst_p = dst + (ptrdiff_t)w8 * d; + uintptr_t temp; + asm volatile( + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "lea (%[src],%[s],2),%[temp] \n" + "vmovdqu (%[temp]),%%xmm2 \n" + "vmovdqu (%[temp],%[s]),%%xmm3 \n" + + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" + + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm3 \n" + + "vmovdqu %%xmm0,(%[dst]) \n" + "vmovdqu %%xmm1,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[temp] \n" + "vmovdqu %%xmm2,(%[temp]) \n" + "vmovdqu %%xmm3,(%[temp],%[d]) \n" + : [temp] "=&r"(temp) + : [src] "r"(src_p), + [dst] "r"(dst_p), + [s] "r"(s), + [d] "r"(d) + : "memory", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); + } + if (rem_w > 0) { + TransposeRemWx4_32_AVX2(src + (ptrdiff_t)w4 * 4, s, + dst + (ptrdiff_t)w4 * d, d, rem_w); + } + src += 4 * s; + dst += 16; + height -= 4; + } + + // Tier 3: 1..3 remaining source rows using 4 x height XMM tiles + corner. + if (height > 0) { + if (w4 > 0) { + const uint8_t* src_p = src; + uint8_t* dst_p = dst; + int cnt = w4; + if (height == 1) { + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "add $0x10,%[src] \n" + "vmovd %%xmm0,(%[dst]) \n" + "vpextrd $1,%%xmm0,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vpextrd $2,%%xmm0,(%[dst]) \n" + "vpextrd $3,%%xmm0,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x4,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(src_p), + [dst] "+r"(dst_p), + [cnt] "+r"(cnt) + : [d] "r"(d) + : "memory", "cc", "xmm0"); + } else if (height == 2) { + asm volatile( + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "add $0x10,%[src] \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm2 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm3 \n" + "vmovq %%xmm2,(%[dst]) \n" + "vmovhpd %%xmm2,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovq %%xmm3,(%[dst]) \n" + "vmovhpd %%xmm3,(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x4,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(src_p), + [dst] "+r"(dst_p), + [cnt] "+r"(cnt) + : [s] "r"(s), + [d] "r"(d) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + } else { + asm volatile( + "vpxor %%xmm3,%%xmm3,%%xmm3 \n" + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vmovdqu (%[src],%[s]),%%xmm1 \n" + "vmovdqu (%[src],%[s],2),%%xmm2 \n" + "add $0x10,%[src] \n" + "vpunpckldq %%xmm1,%%xmm0,%%xmm4 \n" + "vpunpckldq %%xmm3,%%xmm2,%%xmm5 \n" + "vpunpckhdq %%xmm1,%%xmm0,%%xmm6 \n" + "vpunpckhdq %%xmm3,%%xmm2,%%xmm7 \n" + "vpunpcklqdq %%xmm5,%%xmm4,%%xmm0 \n" + "vpunpckhqdq %%xmm5,%%xmm4,%%xmm1 \n" + "vpunpcklqdq %%xmm7,%%xmm6,%%xmm2 \n" + "vmovq %%xmm0,(%[dst]) \n" + "vpextrd $2,%%xmm0,8(%[dst]) \n" + "vmovq %%xmm1,(%[dst],%[d]) \n" + "vpextrd $2,%%xmm1,8(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "vmovq %%xmm2,(%[dst]) \n" + "vpextrd $2,%%xmm2,8(%[dst]) \n" + "vpunpckhqdq %%xmm7,%%xmm6,%%xmm4 \n" + "vmovq %%xmm4,(%[dst],%[d]) \n" + "vpextrd $2,%%xmm4,8(%[dst],%[d]) \n" + "lea (%[dst],%[d],2),%[dst] \n" + "sub $0x4,%[cnt] \n" + "jg 1b \n" + : [src] "+r"(src_p), + [dst] "+r"(dst_p), + [cnt] "+r"(cnt) + : [s] "r"(s), + [d] "r"(d) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + } + } + for (int x = w4; x < width; ++x) { + uint8_t* dst_row = dst + (ptrdiff_t)x * d; + for (int y = 0; y < height; ++y) { + memcpy(dst_row + y * 4, src + (ptrdiff_t)y * s + x * 4, 4); + } + } + } +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX2) + #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) // Transpose 16 rows of 32 bit plane (arbitrary width) using 4x16 ZMM vertical // tiles in zmm0..zmm7 with k1-masked right tail. @@ -1318,8 +1702,8 @@ void TransposeWx16_32_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEWX16_32_AVX512BW) -#endif // !defined(LIBYUV_DISABLE_X86) && defined(__x86_64__) && - // !defined(LIBYUV_ENABLE_ROWWIN) +#endif // !defined(LIBYUV_DISABLE_X86) && (defined(__x86_64__) || + // defined(__i386__)) && !defined(LIBYUV_ENABLE_ROWWIN) #ifdef __cplusplus } // extern "C" diff --git a/source/rotate_win.cc b/source/rotate_win.cc index 707286a0f..d1d48dfea 100644 --- a/source/rotate_win.cc +++ b/source/rotate_win.cc @@ -11,16 +11,17 @@ #include "libyuv/rotate_row.h" #include "libyuv/row.h" -// This module is for 32 bit x86 with any compiler, Visual C 64 bit, and for -// clang with LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. -#if !defined(LIBYUV_DISABLE_X86) && \ - (defined(__i386__) || defined(_M_IX86) || \ - ((defined(__x86_64__) || defined(_M_X64)) && \ - ((defined(_MSC_VER) && !defined(__clang__)) || \ - defined(LIBYUV_ENABLE_ROWWIN)))) +// This module is for Visual C 32/64 bit, and for clang with +// LIBYUV_ENABLE_ROWWIN, which is also enabled for MemorySanitizer. +#if !defined(LIBYUV_DISABLE_X86) && \ + (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || \ + defined(_M_IX86)) && \ + ((defined(_MSC_VER) && !defined(__clang__)) || \ + defined(LIBYUV_ENABLE_ROWWIN)) #include #include // For AVX2 and AVX512 intrinsics +#include #if defined(_MSC_VER) && !defined(__clang__) #include #endif @@ -33,22 +34,21 @@ extern "C" { // Target attributes allow gcc and clang to build each function for its // instruction set regardless of the baseline, which may lack SSE2 for 32 bit. #if defined(__clang__) || defined(__GNUC__) -#define LIBYUV_TARGET_SSE2 __attribute__((target("sse2"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) #else -#define LIBYUV_TARGET_SSE2 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #endif // defined(__clang__) || defined(__GNUC__) -#if defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2) -// Transpose 8 rows x 16 bytes with SSE2. Each output holds 2 columns x 8 rows. -// The low 8 bytes (even column) go to dst_a and the high 8 bytes (odd column) -// go to dst_b. Width is a multiple of 16 bytes. -LIBYUV_TARGET_SSE2 -static void TransposeWx8_Byte_SSE2(const uint8_t* src, +#if defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2) +// Transpose 8 rows x 16 bytes using 2 lanes of AVX2. Source row r is in +// lane r / 4 of register r % 4. Each 128-bit output lane holds 2 columns x 8 +// rows: low 8 bytes (even column) for dst_a and high 8 bytes (odd column) for +// dst_b. Width is a multiple of 16 bytes. +LIBYUV_TARGET_AVX2 +static void TransposeWx8_Byte_AVX2(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, @@ -59,97 +59,101 @@ static void TransposeWx8_Byte_SSE2(const uint8_t* src, const ptrdiff_t sa = dst_stride_a; const ptrdiff_t sb = dst_stride_b; while (byte_width > 0) { - __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); - __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); - __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); - __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); - __m128i r4 = _mm_loadu_si128((const __m128i*)(src + s * 4)); - __m128i r5 = _mm_loadu_si128((const __m128i*)(src + s * 5)); - __m128i r6 = _mm_loadu_si128((const __m128i*)(src + s * 6)); - __m128i r7 = _mm_loadu_si128((const __m128i*)(src + s * 7)); + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 5)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 6)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 7)), 1); - // Bytes of row pairs. a0 is columns 0..7 of rows 0, 1. - __m128i a0 = _mm_unpacklo_epi8(r0, r1); - __m128i a1 = _mm_unpackhi_epi8(r0, r1); - __m128i a2 = _mm_unpacklo_epi8(r2, r3); - __m128i a3 = _mm_unpackhi_epi8(r2, r3); - __m128i a4 = _mm_unpacklo_epi8(r4, r5); - __m128i a5 = _mm_unpackhi_epi8(r4, r5); - __m128i a6 = _mm_unpacklo_epi8(r6, r7); - __m128i a7 = _mm_unpackhi_epi8(r6, r7); + __m256i a0 = _mm256_unpacklo_epi8(r0, r1); + __m256i a1 = _mm256_unpackhi_epi8(r0, r1); + __m256i a2 = _mm256_unpacklo_epi8(r2, r3); + __m256i a3 = _mm256_unpackhi_epi8(r2, r3); - // Words. b0 is columns 0..3 of rows 0..3. - __m128i b0 = _mm_unpacklo_epi16(a0, a2); - __m128i b1 = _mm_unpackhi_epi16(a0, a2); - __m128i b2 = _mm_unpacklo_epi16(a1, a3); - __m128i b3 = _mm_unpackhi_epi16(a1, a3); - __m128i b4 = _mm_unpacklo_epi16(a4, a6); - __m128i b5 = _mm_unpackhi_epi16(a4, a6); - __m128i b6 = _mm_unpacklo_epi16(a5, a7); - __m128i b7 = _mm_unpackhi_epi16(a5, a7); + __m256i b0 = _mm256_unpacklo_epi16(a0, a2); + __m256i b1 = _mm256_unpackhi_epi16(a0, a2); + __m256i b2 = _mm256_unpacklo_epi16(a1, a3); + __m256i b3 = _mm256_unpackhi_epi16(a1, a3); - // Dwords. c0 is columns 0, 1 of rows 0..7. - __m128i c0 = _mm_unpacklo_epi32(b0, b4); - __m128i c1 = _mm_unpackhi_epi32(b0, b4); - __m128i c2 = _mm_unpacklo_epi32(b1, b5); - __m128i c3 = _mm_unpackhi_epi32(b1, b5); - __m128i c4 = _mm_unpacklo_epi32(b2, b6); - __m128i c5 = _mm_unpackhi_epi32(b2, b6); - __m128i c6 = _mm_unpacklo_epi32(b3, b7); - __m128i c7 = _mm_unpackhi_epi32(b3, b7); + __m256i t0 = _mm256_permute2x128_si256(b0, b1, 0x20); + __m256i t1 = _mm256_permute2x128_si256(b0, b1, 0x31); + __m256i t2 = _mm256_permute2x128_si256(b2, b3, 0x20); + __m256i t3 = _mm256_permute2x128_si256(b2, b3, 0x31); - _mm_storel_epi64((__m128i*)(dst_a), c0); - _mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(c0)); - _mm_storel_epi64((__m128i*)(dst_a + sa), c1); - _mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(c1)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 2), c2); - _mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(c2)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 3), c3); - _mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(c3)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 4), c4); - _mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(c4)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 5), c5); - _mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(c5)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 6), c6); - _mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(c6)); - _mm_storel_epi64((__m128i*)(dst_a + sa * 7), c7); - _mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(c7)); + __m256i c0 = _mm256_unpacklo_epi32(t0, t1); + __m256i c1 = _mm256_unpackhi_epi32(t0, t1); + __m256i c2 = _mm256_unpacklo_epi32(t2, t3); + __m256i c3 = _mm256_unpackhi_epi32(t2, t3); + + __m128i lo0 = _mm256_castsi256_si128(c0); + __m128i lo1 = _mm256_castsi256_si128(c1); + __m128i hi0 = _mm256_extracti128_si256(c0, 1); + __m128i hi1 = _mm256_extracti128_si256(c1, 1); + __m128i lo2 = _mm256_castsi256_si128(c2); + __m128i lo3 = _mm256_castsi256_si128(c3); + __m128i hi2 = _mm256_extracti128_si256(c2, 1); + __m128i hi3 = _mm256_extracti128_si256(c3, 1); + + _mm_storel_epi64((__m128i*)(dst_a), lo0); + _mm_storeh_pd((double*)(dst_b), _mm_castsi128_pd(lo0)); + _mm_storel_epi64((__m128i*)(dst_a + sa), lo1); + _mm_storeh_pd((double*)(dst_b + sb), _mm_castsi128_pd(lo1)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 2), hi0); + _mm_storeh_pd((double*)(dst_b + sb * 2), _mm_castsi128_pd(hi0)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 3), hi1); + _mm_storeh_pd((double*)(dst_b + sb * 3), _mm_castsi128_pd(hi1)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 4), lo2); + _mm_storeh_pd((double*)(dst_b + sb * 4), _mm_castsi128_pd(lo2)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 5), lo3); + _mm_storeh_pd((double*)(dst_b + sb * 5), _mm_castsi128_pd(lo3)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 6), hi2); + _mm_storeh_pd((double*)(dst_b + sb * 6), _mm_castsi128_pd(hi2)); + _mm_storel_epi64((__m128i*)(dst_a + sa * 7), hi3); + _mm_storeh_pd((double*)(dst_b + sb * 7), _mm_castsi128_pd(hi3)); src += 16; dst_a += sa * 8; dst_b += sb * 8; byte_width -= 16; } + _mm256_zeroupper(); } -#endif // defined(HAS_TRANSPOSEWX8_SSSE3) || defined(HAS_TRANSPOSEUVWX8_SSE2) +#endif // defined(HAS_TRANSPOSEWX8_AVX2) || defined(HAS_TRANSPOSEUVWX8_AVX2) -#if defined(HAS_TRANSPOSEWX8_SSSE3) +#if defined(HAS_TRANSPOSEWX8_AVX2) // Transpose 16x8. Width is a multiple of 16. -LIBYUV_TARGET_SSE2 -void TransposeWx8_SSSE3(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - TransposeWx8_Byte_SSE2(src, src_stride, dst, dst_stride * 2, dst + dst_stride, +LIBYUV_TARGET_AVX2 +void TransposeWx8_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + TransposeWx8_Byte_AVX2(src, src_stride, dst, dst_stride * 2, dst + dst_stride, dst_stride * 2, width); } -#endif // defined(HAS_TRANSPOSEWX8_SSSE3) +#endif // defined(HAS_TRANSPOSEWX8_AVX2) -#if defined(HAS_TRANSPOSEUVWX8_SSE2) +#if defined(HAS_TRANSPOSEUVWX8_AVX2) // Transpose UV 8x8. Width is a multiple of 8. -LIBYUV_TARGET_SSE2 -void TransposeUVWx8_SSE2(const uint8_t* src, +LIBYUV_TARGET_AVX2 +void TransposeUVWx8_AVX2(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width) { - TransposeWx8_Byte_SSE2(src, src_stride, dst_a, dst_stride_a, dst_b, + TransposeWx8_Byte_AVX2(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, width * 2); } -#endif // defined(HAS_TRANSPOSEUVWX8_SSE2) +#endif // defined(HAS_TRANSPOSEUVWX8_AVX2) #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEUVWX16_AVX2) // Load row r into lane 0 and row r + 8 into lane 1. @@ -417,6 +421,55 @@ void TransposeUVWx16_AVX512BW(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEUVWX16_AVX512BW) +#if defined(HAS_TRANSPOSEWX8_16_AVX2) +LIBYUV_TARGET_AVX2 +void TransposeWx8_16_AVX2(const uint16_t* src, + int src_stride, + uint16_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + while (width > 0) { + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 5)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 6)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 7)), 1); + + __m256i t0 = _mm256_unpacklo_epi16(r0, r1); + __m256i t1 = _mm256_unpackhi_epi16(r0, r1); + __m256i t2 = _mm256_unpacklo_epi16(r2, r3); + __m256i t3 = _mm256_unpackhi_epi16(r2, r3); + + __m256i u0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(t0, t2), 0xd8); + __m256i u1 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(t0, t2), 0xd8); + __m256i u2 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(t1, t3), 0xd8); + __m256i u3 = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(t1, t3), 0xd8); + + _mm_storeu_si128((__m128i*)(dst), _mm256_castsi256_si128(u0)); + _mm_storeu_si128((__m128i*)(dst + d), _mm256_extracti128_si256(u0, 1)); + _mm_storeu_si128((__m128i*)(dst + d * 2), _mm256_castsi256_si128(u1)); + _mm_storeu_si128((__m128i*)(dst + d * 3), _mm256_extracti128_si256(u1, 1)); + _mm_storeu_si128((__m128i*)(dst + d * 4), _mm256_castsi256_si128(u2)); + _mm_storeu_si128((__m128i*)(dst + d * 5), _mm256_extracti128_si256(u2, 1)); + _mm_storeu_si128((__m128i*)(dst + d * 6), _mm256_castsi256_si128(u3)); + _mm_storeu_si128((__m128i*)(dst + d * 7), _mm256_extracti128_si256(u3, 1)); + src += 8; + dst += d * 8; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // defined(HAS_TRANSPOSEWX8_16_AVX2) + #if defined(HAS_TRANSPOSEWX16_16_AVX2) LIBYUV_TARGET_AVX2 void TransposeWx16_16_AVX2(const uint16_t* src, @@ -709,35 +762,61 @@ void TransposeWx32_16_AVX512BW(const uint16_t* src, } #endif // defined(HAS_TRANSPOSEWX32_16_AVX512BW) -#if defined(HAS_TRANSPOSEWX4_32_SSE2) -// Transpose 32 bit values (ARGB). Read a column of 4 pixels, write a row. -LIBYUV_TARGET_SSE2 -void TransposeWx4_32_SSE2(const uint8_t* src, +#if defined(HAS_TRANSPOSEWX4_32_AVX2) +// Transpose 32 bit values (ARGB). Read a column of 4 pixels across 8 rows per +// AVX2 iteration (with a 4-row 128-bit tail) and write 4 destination rows. +LIBYUV_TARGET_AVX2 +void TransposeWx4_32_AVX2(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) { const ptrdiff_t s = src_stride; const ptrdiff_t d = dst_stride; - while (width > 0) { - __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); // a b c d - __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); // e f g h - __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); // i j k l - __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); // m n o p - __m128i t0 = _mm_unpacklo_epi32(r0, r1); // a e b f - __m128i t1 = _mm_unpacklo_epi32(r2, r3); // i m j n - __m128i t2 = _mm_unpackhi_epi32(r0, r1); // c g d h - __m128i t3 = _mm_unpackhi_epi32(r2, r3); // k o l p - _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); // a e i m + while (width >= 8) { + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src))), + _mm_loadu_si128((const __m128i*)(src + s * 4)), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s))), + _mm_loadu_si128((const __m128i*)(src + s * 5)), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 2))), + _mm_loadu_si128((const __m128i*)(src + s * 6)), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_loadu_si128((const __m128i*)(src + s * 3))), + _mm_loadu_si128((const __m128i*)(src + s * 7)), 1); + + __m256i t0 = _mm256_unpacklo_epi32(r0, r1); + __m256i t1 = _mm256_unpacklo_epi32(r2, r3); + __m256i t2 = _mm256_unpackhi_epi32(r0, r1); + __m256i t3 = _mm256_unpackhi_epi32(r2, r3); + + _mm256_storeu_si256((__m256i*)(dst), _mm256_unpacklo_epi64(t0, t1)); + _mm256_storeu_si256((__m256i*)(dst + d), _mm256_unpackhi_epi64(t0, t1)); + _mm256_storeu_si256((__m256i*)(dst + d * 2), _mm256_unpacklo_epi64(t2, t3)); + _mm256_storeu_si256((__m256i*)(dst + d * 3), _mm256_unpackhi_epi64(t2, t3)); + src += s * 8; + dst += 32; + width -= 8; + } + if (width > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src + s * 2)); + __m128i r3 = _mm_loadu_si128((const __m128i*)(src + s * 3)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst), _mm_unpacklo_epi64(t0, t1)); _mm_storeu_si128((__m128i*)(dst + d), _mm_unpackhi_epi64(t0, t1)); _mm_storeu_si128((__m128i*)(dst + d * 2), _mm_unpacklo_epi64(t2, t3)); _mm_storeu_si128((__m128i*)(dst + d * 3), _mm_unpackhi_epi64(t2, t3)); - src += s * 4; - dst += 16; - width -= 4; } + _mm256_zeroupper(); } -#endif // defined(HAS_TRANSPOSEWX4_32_SSE2) +#endif // defined(HAS_TRANSPOSEWX4_32_AVX2) #if defined(HAS_TRANSPOSEWX8_32_AVX2) LIBYUV_TARGET_AVX2 @@ -779,6 +858,241 @@ void TransposeWx8_32_AVX2(const uint8_t* src, } #endif // defined(HAS_TRANSPOSEWX8_32_AVX2) +#if defined(HAS_TRANSPOSEWXH_32_AVX2) +// Load n (1..3) 32-bit pixels into low lanes of __m128i with zeroed upper +// lanes (safe at buffer boundaries and MemorySanitizer-clean). +LIBYUV_TARGET_AVX2 +static __m128i LoadPartialRow_32_AVX2(const uint8_t* p, int n) { + if (n == 1) { + int v; + memcpy(&v, p, 4); + return _mm_cvtsi32_si128(v); + } + if (n == 2) { + return _mm_loadl_epi64((const __m128i*)p); + } + int v; + memcpy(&v, p + 8, 4); + return _mm_insert_epi32(_mm_loadl_epi64((const __m128i*)p), v, 2); +} + +// Transpose arbitrary WxH 32-bit plane using AVX2 (4x8, 8x4, 4x4, and 4x{1..3} +// tiles plus exact remainder handling). +LIBYUV_TARGET_AVX2 +void TransposeWxH_32_AVX2(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width, + int height) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const int w4 = width & ~3; + const int rem_w = width & 3; + + // Tier 1: 8-row strips using 4x8 YMM tiles + rem_w x 8 YMM tail. + while (height >= 8) { + if (w4 > 0) { + TransposeWx8_32_AVX2(src, src_stride, dst, dst_stride, w4); + } + if (rem_w > 0) { + const uint8_t* src_rem = src + (ptrdiff_t)w4 * 4; + uint8_t* dst_rem = dst + (ptrdiff_t)w4 * d; + __m256i r0 = _mm256_inserti128_si256( + _mm256_castsi128_si256(LoadPartialRow_32_AVX2(src_rem, rem_w)), + LoadPartialRow_32_AVX2(src_rem + s * 4, rem_w), 1); + __m256i r1 = _mm256_inserti128_si256( + _mm256_castsi128_si256(LoadPartialRow_32_AVX2(src_rem + s, rem_w)), + LoadPartialRow_32_AVX2(src_rem + s * 5, rem_w), 1); + __m256i r2 = _mm256_inserti128_si256( + _mm256_castsi128_si256( + LoadPartialRow_32_AVX2(src_rem + s * 2, rem_w)), + LoadPartialRow_32_AVX2(src_rem + s * 6, rem_w), 1); + __m256i r3 = _mm256_inserti128_si256( + _mm256_castsi128_si256( + LoadPartialRow_32_AVX2(src_rem + s * 3, rem_w)), + LoadPartialRow_32_AVX2(src_rem + s * 7, rem_w), 1); + + __m256i t0 = _mm256_unpacklo_epi32(r0, r1); + __m256i t1 = _mm256_unpacklo_epi32(r2, r3); + _mm256_storeu_si256((__m256i*)(dst_rem), _mm256_unpacklo_epi64(t0, t1)); + if (rem_w > 1) { + _mm256_storeu_si256((__m256i*)(dst_rem + d), + _mm256_unpackhi_epi64(t0, t1)); + } + if (rem_w > 2) { + __m256i t2 = _mm256_unpackhi_epi32(r0, r1); + __m256i t3 = _mm256_unpackhi_epi32(r2, r3); + _mm256_storeu_si256((__m256i*)(dst_rem + d * 2), + _mm256_unpacklo_epi64(t2, t3)); + } + _mm256_zeroupper(); + } + src += 8 * s; + dst += 32; + height -= 8; + } + + // Tier 2: 4-row strip using 8x4 YMM tiles + 4x4 XMM block + rem_w x 4 tail. + if (height >= 4) { + const int w8 = width & ~7; + if (w8 > 0) { + const uint8_t* src_p = src; + uint8_t* dst_p = dst; + int cnt = w8; + while (cnt > 0) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)(src_p)); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_p + s)); + __m256i r2 = _mm256_loadu_si256((const __m256i*)(src_p + s * 2)); + __m256i r3 = _mm256_loadu_si256((const __m256i*)(src_p + s * 3)); + + __m256i t0 = _mm256_unpacklo_epi32(r0, r1); + __m256i t1 = _mm256_unpacklo_epi32(r2, r3); + __m256i t2 = _mm256_unpackhi_epi32(r0, r1); + __m256i t3 = _mm256_unpackhi_epi32(r2, r3); + + __m256i o0 = _mm256_unpacklo_epi64(t0, t1); + __m256i o1 = _mm256_unpackhi_epi64(t0, t1); + __m256i o2 = _mm256_unpacklo_epi64(t2, t3); + __m256i o3 = _mm256_unpackhi_epi64(t2, t3); + + _mm_storeu_si128((__m128i*)(dst_p), _mm256_castsi256_si128(o0)); + _mm_storeu_si128((__m128i*)(dst_p + d), _mm256_castsi256_si128(o1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 2), _mm256_castsi256_si128(o2)); + _mm_storeu_si128((__m128i*)(dst_p + d * 3), _mm256_castsi256_si128(o3)); + _mm_storeu_si128((__m128i*)(dst_p + d * 4), + _mm256_extracti128_si256(o0, 1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 5), + _mm256_extracti128_si256(o1, 1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 6), + _mm256_extracti128_si256(o2, 1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 7), + _mm256_extracti128_si256(o3, 1)); + + src_p += 32; + dst_p += d * 8; + cnt -= 8; + } + _mm256_zeroupper(); + } + if (width & 4) { + const uint8_t* src_p = src + (ptrdiff_t)w8 * 4; + uint8_t* dst_p = dst + (ptrdiff_t)w8 * d; + __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src_p + s * 2)); + __m128i r3 = _mm_loadu_si128((const __m128i*)(src_p + s * 3)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst_p), _mm_unpacklo_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst_p + d), _mm_unpackhi_epi64(t0, t1)); + _mm_storeu_si128((__m128i*)(dst_p + d * 2), _mm_unpacklo_epi64(t2, t3)); + _mm_storeu_si128((__m128i*)(dst_p + d * 3), _mm_unpackhi_epi64(t2, t3)); + } + if (rem_w > 0) { + const uint8_t* src_rem = src + (ptrdiff_t)w4 * 4; + uint8_t* dst_rem = dst + (ptrdiff_t)w4 * d; + __m128i r0 = LoadPartialRow_32_AVX2(src_rem, rem_w); + __m128i r1 = LoadPartialRow_32_AVX2(src_rem + s, rem_w); + __m128i r2 = LoadPartialRow_32_AVX2(src_rem + s * 2, rem_w); + __m128i r3 = LoadPartialRow_32_AVX2(src_rem + s * 3, rem_w); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst_rem), _mm_unpacklo_epi64(t0, t1)); + if (rem_w > 1) { + _mm_storeu_si128((__m128i*)(dst_rem + d), _mm_unpackhi_epi64(t0, t1)); + } + if (rem_w > 2) { + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, r3); + _mm_storeu_si128((__m128i*)(dst_rem + d * 2), + _mm_unpacklo_epi64(t2, t3)); + } + } + src += 4 * s; + dst += 16; + height -= 4; + } + + // Tier 3: 1..3 remaining source rows using 4 x height XMM tiles + corner. + if (height > 0) { + if (w4 > 0) { + const uint8_t* src_p = src; + uint8_t* dst_p = dst; + int cnt = w4; + if (height == 1) { + while (cnt > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p)); + int v0 = _mm_cvtsi128_si32(r0); + int v1 = _mm_extract_epi32(r0, 1); + int v2 = _mm_extract_epi32(r0, 2); + int v3 = _mm_extract_epi32(r0, 3); + memcpy(dst_p, &v0, 4); + memcpy(dst_p + d, &v1, 4); + memcpy(dst_p + d * 2, &v2, 4); + memcpy(dst_p + d * 3, &v3, 4); + src_p += 16; + dst_p += d * 4; + cnt -= 4; + } + } else if (height == 2) { + while (cnt > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpackhi_epi32(r0, r1); + _mm_storel_epi64((__m128i*)(dst_p), t0); + _mm_storeh_pd((double*)(dst_p + d), _mm_castsi128_pd(t0)); + _mm_storel_epi64((__m128i*)(dst_p + d * 2), t1); + _mm_storeh_pd((double*)(dst_p + d * 3), _mm_castsi128_pd(t1)); + src_p += 16; + dst_p += d * 4; + cnt -= 4; + } + } else { + const __m128i zero = _mm_setzero_si128(); + while (cnt > 0) { + __m128i r0 = _mm_loadu_si128((const __m128i*)(src_p)); + __m128i r1 = _mm_loadu_si128((const __m128i*)(src_p + s)); + __m128i r2 = _mm_loadu_si128((const __m128i*)(src_p + s * 2)); + __m128i t0 = _mm_unpacklo_epi32(r0, r1); + __m128i t1 = _mm_unpacklo_epi32(r2, zero); + __m128i t2 = _mm_unpackhi_epi32(r0, r1); + __m128i t3 = _mm_unpackhi_epi32(r2, zero); + __m128i o0 = _mm_unpacklo_epi64(t0, t1); + __m128i o1 = _mm_unpackhi_epi64(t0, t1); + __m128i o2 = _mm_unpacklo_epi64(t2, t3); + __m128i o3 = _mm_unpackhi_epi64(t2, t3); + int v0 = _mm_extract_epi32(o0, 2); + int v1 = _mm_extract_epi32(o1, 2); + int v2 = _mm_extract_epi32(o2, 2); + int v3 = _mm_extract_epi32(o3, 2); + _mm_storel_epi64((__m128i*)(dst_p), o0); + memcpy(dst_p + 8, &v0, 4); + _mm_storel_epi64((__m128i*)(dst_p + d), o1); + memcpy(dst_p + d + 8, &v1, 4); + _mm_storel_epi64((__m128i*)(dst_p + d * 2), o2); + memcpy(dst_p + d * 2 + 8, &v2, 4); + _mm_storel_epi64((__m128i*)(dst_p + d * 3), o3); + memcpy(dst_p + d * 3 + 8, &v3, 4); + src_p += 16; + dst_p += d * 4; + cnt -= 4; + } + } + } + for (int x = w4; x < width; ++x) { + uint8_t* dst_row = dst + (ptrdiff_t)x * d; + for (int y = 0; y < height; ++y) { + memcpy(dst_row + y * 4, src + (ptrdiff_t)y * s + x * 4, 4); + } + } + } +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX2) + #if defined(HAS_TRANSPOSEWX16_32_AVX512BW) LIBYUV_TARGET_AVX512BW void TransposeWx16_32_AVX512BW(const uint8_t* src, diff --git a/source/row_any.cc b/source/row_any.cc index bb59eaf3e..3ac640620 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -277,7 +277,7 @@ ANY41PT(MergeARGB16To8Row_Any_NEON, const uint8_t* v_buf, uint8_t* dst_ptr, int width) { \ SIMD_ALIGNED(uint8_t vin[64 * 3]); \ static_assert(MASK + 1 <= sizeof(vin) / 3, "vin buffer too small"); \ - SIMD_ALIGNED(uint8_t vout[64]); \ + SIMD_ALIGNED(uint8_t vout[64 * 4]); \ static_assert(SS(MASK + 1, DUVSHIFT) * BPP <= sizeof(vout), \ "vout buffer too small"); \ memset(vin, 0, sizeof(vin)); /* for YUY2 and msan */ \ @@ -296,8 +296,8 @@ ANY41PT(MergeARGB16To8Row_Any_NEON, } // Merge functions. -#ifdef HAS_MERGERGBROW_SSSE3 -ANY31(MergeRGBRow_Any_SSSE3, MergeRGBRow_SSSE3, 0, 0, 3, 15) +#ifdef HAS_MERGERGBROW_AVX2 +ANY31(MergeRGBRow_Any_AVX2, MergeRGBRow_AVX2, 0, 0, 3, 31) #endif #ifdef HAS_MERGERGBROW_NEON ANY31(MergeRGBRow_Any_NEON, MergeRGBRow_NEON, 0, 0, 3, 15) @@ -769,8 +769,8 @@ ANY21(ARGBAlphaBlendRow_Any_LASX, ARGBAlphaBlendRow_LASX, 0, 4, 4, 4, 7) #ifdef HAS_ARGBALPHABLENDROW_WASMSIMD ANY21(ARGBAlphaBlendRow_Any_WASMSIMD, ARGBAlphaBlendRow_WASMSIMD, 0, 4, 4, 4, 3) #endif -#ifdef HAS_SOBELROW_SSE2 -ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15) +#ifdef HAS_SOBELROW_AVX2 +ANY21(SobelRow_Any_AVX2, SobelRow_AVX2, 0, 1, 1, 4, 15) #endif #ifdef HAS_SOBELROW_NEON ANY21(SobelRow_Any_NEON, SobelRow_NEON, 0, 1, 1, 4, 7) @@ -781,8 +781,8 @@ ANY21(SobelRow_Any_LSX, SobelRow_LSX, 0, 1, 1, 4, 7) #ifdef HAS_SOBELROW_LASX ANY21(SobelRow_Any_LASX, SobelRow_LASX, 0, 1, 1, 4, 15) #endif -#ifdef HAS_SOBELTOPLANEROW_SSE2 -ANY21(SobelToPlaneRow_Any_SSE2, SobelToPlaneRow_SSE2, 0, 1, 1, 1, 15) +#ifdef HAS_SOBELTOPLANEROW_AVX2 +ANY21(SobelToPlaneRow_Any_AVX2, SobelToPlaneRow_AVX2, 0, 1, 1, 1, 31) #endif #ifdef HAS_SOBELTOPLANEROW_NEON ANY21(SobelToPlaneRow_Any_NEON, SobelToPlaneRow_NEON, 0, 1, 1, 1, 15) @@ -793,8 +793,8 @@ ANY21(SobelToPlaneRow_Any_LSX, SobelToPlaneRow_LSX, 0, 1, 1, 1, 15) #ifdef HAS_SOBELTOPLANEROW_LASX ANY21(SobelToPlaneRow_Any_LASX, SobelToPlaneRow_LASX, 0, 1, 1, 1, 31) #endif -#ifdef HAS_SOBELXYROW_SSE2 -ANY21(SobelXYRow_Any_SSE2, SobelXYRow_SSE2, 0, 1, 1, 4, 15) +#ifdef HAS_SOBELXYROW_AVX2 +ANY21(SobelXYRow_Any_AVX2, SobelXYRow_AVX2, 0, 1, 1, 4, 15) #endif #ifdef HAS_SOBELXYROW_NEON ANY21(SobelXYRow_Any_NEON, SobelXYRow_NEON, 0, 1, 1, 4, 7) @@ -1114,6 +1114,15 @@ ANY11(AR30ToABGRRow_Any_NEON, AR30ToABGRRow_NEON, 0, 4, 4, 7) #if defined(HAS_AR30TOAB30ROW_NEON) ANY11(AR30ToAB30Row_Any_NEON, AR30ToAB30Row_NEON, 0, 4, 4, 7) #endif +#if defined(HAS_AR30TOARGBROW_AVX2) +ANY11(AR30ToARGBRow_Any_AVX2, AR30ToARGBRow_AVX2, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOABGRROW_AVX2) +ANY11(AR30ToABGRRow_Any_AVX2, AR30ToABGRRow_AVX2, 0, 4, 4, 7) +#endif +#if defined(HAS_AR30TOAB30ROW_AVX2) +ANY11(AR30ToAB30Row_Any_AVX2, AR30ToAB30Row_AVX2, 0, 4, 4, 7) +#endif #if defined(HAS_ARGBTOAR30ROW_SSSE3) ANY11(ARGBToAR30Row_Any_SSSE3, ARGBToAR30Row_SSSE3, 0, 4, 4, 3) #endif @@ -1141,8 +1150,8 @@ ANY11(RAWToARGBRow_Any_AVX2, RAWToARGBRow_AVX2, 0, 3, 4, 31) #if defined(HAS_RGB24TOARGBROW_AVX2) ANY11(RGB24ToARGBRow_Any_AVX2, RGB24ToARGBRow_AVX2, 0, 3, 4, 31) #endif -#if defined(HAS_RAWTORGBAROW_SSSE3) -ANY11(RAWToRGBARow_Any_SSSE3, RAWToRGBARow_SSSE3, 0, 3, 4, 15) +#if defined(HAS_RAWTORGBAROW_AVX2) +ANY11(RAWToRGBARow_Any_AVX2, RAWToRGBARow_AVX2, 0, 3, 4, 15) #endif #if defined(HAS_RAWTORGB24ROW_SSSE3) ANY11(RAWToRGB24Row_Any_SSSE3, RAWToRGB24Row_SSSE3, 0, 3, 3, 7) @@ -1217,6 +1226,15 @@ ANY11(YUY2ToYRow_Any_SSE2, YUY2ToYRow_SSE2, 1, 4, 1, 15) ANY11(UYVYToYRow_Any_SSE2, UYVYToYRow_SSE2, 1, 4, 1, 15) #endif +#ifdef HAS_RGB565TOYROW_AVX2 +ANY11(RGB565ToYRow_Any_AVX2, RGB565ToYRow_AVX2, 0, 2, 1, 31) +#endif +#ifdef HAS_ARGB1555TOYROW_AVX2 +ANY11(ARGB1555ToYRow_Any_AVX2, ARGB1555ToYRow_AVX2, 0, 2, 1, 31) +#endif +#ifdef HAS_ARGB4444TOYROW_AVX2 +ANY11(ARGB4444ToYRow_Any_AVX2, ARGB4444ToYRow_AVX2, 0, 2, 1, 31) +#endif #ifdef HAS_RGB565TOYROW_NEON ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 15) #endif @@ -1264,6 +1282,9 @@ ANY11(UYVYToYRow_Any_LSX, UYVYToYRow_LSX, 1, 4, 1, 15) #ifdef HAS_UYVYTOYROW_LASX ANY11(UYVYToYRow_Any_LASX, UYVYToYRow_LASX, 1, 4, 1, 31) #endif +#ifdef HAS_AYUVTOYROW_AVX2 +ANY11(AYUVToYRow_Any_AVX2, AYUVToYRow_AVX2, 0, 4, 1, 15) +#endif #ifdef HAS_AYUVTOYROW_NEON ANY11(AYUVToYRow_Any_NEON, AYUVToYRow_NEON, 0, 4, 1, 15) #endif @@ -1670,6 +1691,9 @@ ANY11T(AR64ToARGBRow_Any_NEON, AR64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) ANY11T(AB64ToARGBRow_Any_NEON, AB64ToARGBRow_NEON, 8, 4, uint16_t, uint8_t, 7) #endif +#ifdef HAS_COPYROW_16_AVX2 +ANY11T(CopyRow_16_Any_AVX2, CopyRow_16_AVX2, 2, 2, uint16_t, uint16_t, 31) +#endif #ifdef HAS_COPYROW_16_NEON ANY11T(CopyRow_16_Any_NEON, CopyRow_16_NEON, 2, 2, uint16_t, uint16_t, 15) #endif @@ -1898,6 +1922,9 @@ ANY11P16(HalfFloat1Row_Any_F16C, #ifdef HAS_HALFFLOATROW_NEON ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 15) #endif +#ifdef HAS_BYTETOFLOATROW_AVX2 +ANY11P16(ByteToFloatRow_Any_AVX2, ByteToFloatRow_AVX2, uint8_t, float, 1, 4, 15) +#endif #ifdef HAS_BYTETOFLOATROW_NEON ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 4, 7) #endif @@ -2031,6 +2058,39 @@ ANY11I(InterpolateRow_16_Any_WASMSIMD, #endif #undef ANY11I +// Any 1 to 1 half row +#define ANY11H(NAMEANY, ANY_SIMD, T, MASK) \ + void NAMEANY(const T* src_ptr, ptrdiff_t src_stride, T* dst_ptr, \ + int width) { \ + SIMD_ALIGNED(T vin[64 * 2]); \ + static_assert((MASK + 1) * sizeof(T) <= sizeof(vin) / 2, \ + "vin buffer too small"); \ + SIMD_ALIGNED(T vout[64]); \ + static_assert((MASK + 1) * sizeof(T) <= sizeof(vout), \ + "vout buffer too small"); \ + MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ + MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ + int r = width & MASK; \ + int n = width & ~MASK; \ + if (n > 0) { \ + ANY_SIMD(src_ptr, src_stride, dst_ptr, n); \ + } \ + ptrdiff_t np = n; \ + memcpy(vin, src_ptr + np, r * sizeof(T)); \ + memcpy(vin + 64, src_ptr + src_stride + np, r * sizeof(T)); \ + ANY_SIMD(vin, 64, vout, MASK + 1); \ + memcpy(dst_ptr + np, vout, r * sizeof(T)); \ + } + +#ifdef HAS_HALFROW_AVX2 +ANY11H(HalfRow_Any_AVX2, HalfRow_AVX2, uint8_t, 31) +#endif +#ifdef HAS_HALFROW_16_AVX2 +ANY11H(HalfRow_16_Any_AVX2, HalfRow_16_AVX2, uint16_t, 15) +#endif + +#undef ANY11H + // Any 1 to 1 half row with scale param #define ANY11HS(NAMEANY, ANY_SIMD, TS, TD, MASK) \ void NAMEANY(const TS* src_ptr, ptrdiff_t src_stride, TD* dst_ptr, \ @@ -2149,9 +2209,6 @@ ANY11HWS(HalfWidthRow_16To8_Any_NEON, memcpy(dst_ptr + np * BPP, vout + (MASK + 1 - r) * BPP, r * BPP); \ } -#ifdef HAS_MIRRORROW_AVX512BW -ANY11M(MirrorRow_Any_AVX512BW, MirrorRow_AVX512BW, 1, 63) -#endif #ifdef HAS_MIRRORROW_AVX2 ANY11M(MirrorRow_Any_AVX2, MirrorRow_AVX2, 1, 31) #endif @@ -2167,9 +2224,6 @@ ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 15) #ifdef HAS_MIRRORROW_LASX ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 31) #endif -#ifdef HAS_MIRRORUVROW_AVX512BW -ANY11M(MirrorUVRow_Any_AVX512BW, MirrorUVRow_AVX512BW, 2, 31) -#endif #ifdef HAS_MIRRORUVROW_AVX2 ANY11M(MirrorUVRow_Any_AVX2, MirrorUVRow_AVX2, 2, 15) #endif @@ -2264,8 +2318,8 @@ ANY11MT(MirrorRow_16_Any_NEON, MirrorRow_16_NEON, uint16_t, 2, 15) memcpy(dst_ptr + np * BPP, vout, r * BPP); \ } -#ifdef HAS_SETROW_X86 -ANY1(SetRow_Any_X86, SetRow_X86, uint8_t, 1, 3) +#ifdef HAS_SETROW_AVX2 +ANY1(SetRow_Any_AVX2, SetRow_AVX2, uint8_t, 1, 31) #endif #ifdef HAS_SETROW_NEON ANY1(SetRow_Any_NEON, SetRow_NEON, uint8_t, 1, 15) @@ -2276,6 +2330,9 @@ ANY1(SetRow_Any_LSX, SetRow_LSX, uint8_t, 1, 15) #ifdef HAS_SETROW_LASX ANY1(SetRow_Any_LASX, SetRow_LASX, uint8_t, 1, 31) #endif +#ifdef HAS_ARGBSETROW_AVX2 +ANY1(ARGBSetRow_Any_AVX2, ARGBSetRow_AVX2, uint32_t, 4, 7) +#endif #ifdef HAS_ARGBSETROW_NEON ANY1(ARGBSetRow_Any_NEON, ARGBSetRow_NEON, uint32_t, 4, 3) #endif @@ -2788,6 +2845,15 @@ ANY12S(RAWToUVRow_Any_LSX, RAWToUVRow_LSX, 0, 3, 15) #ifdef HAS_RAWTOUVROW_LASX ANY12S(RAWToUVRow_Any_LASX, RAWToUVRow_LASX, 0, 3, 31) #endif +#ifdef HAS_RGB565TOUVROW_AVX2 +ANY12S(RGB565ToUVRow_Any_AVX2, RGB565ToUVRow_AVX2, 0, 2, 31) +#endif +#ifdef HAS_ARGB1555TOUVROW_AVX2 +ANY12S(ARGB1555ToUVRow_Any_AVX2, ARGB1555ToUVRow_AVX2, 0, 2, 31) +#endif +#ifdef HAS_ARGB4444TOUVROW_AVX2 +ANY12S(ARGB4444ToUVRow_Any_AVX2, ARGB4444ToUVRow_AVX2, 0, 2, 31) +#endif #ifdef HAS_RGB565TOUVROW_NEON ANY12S(RGB565ToUVRow_Any_NEON, RGB565ToUVRow_NEON, 0, 2, 15) #endif @@ -2861,6 +2927,12 @@ ANY12S(UYVYToUVRow_Any_LASX, UYVYToUVRow_LASX, 1, 4, 31) memcpy(dst_vu + (np >> 1) * 2, vout, SS(r, 1) * 2); \ } +#ifdef HAS_AYUVTOUVROW_AVX2 +ANY11S(AYUVToUVRow_Any_AVX2, AYUVToUVRow_AVX2, 0, 4, 15) +#endif +#ifdef HAS_AYUVTOVUROW_AVX2 +ANY11S(AYUVToVURow_Any_AVX2, AYUVToVURow_AVX2, 0, 4, 15) +#endif #ifdef HAS_AYUVTOVUROW_NEON ANY11S(AYUVToUVRow_Any_NEON, AYUVToUVRow_NEON, 0, 4, 15) ANY11S(AYUVToVURow_Any_NEON, AYUVToVURow_NEON, 0, 4, 15) @@ -2895,17 +2967,14 @@ ANY11S(AYUVToVURow_Any_SVE2, AYUVToVURow_SVE2, 0, 4, 1) #ifdef HAS_DETILEROW_NEON ANYDETILE(DetileRow_Any_NEON, DetileRow_NEON, uint8_t, 1, 15) #endif -#ifdef HAS_DETILEROW_SSE2 -ANYDETILE(DetileRow_Any_SSE2, DetileRow_SSE2, uint8_t, 1, 15) +#ifdef HAS_DETILEROW_AVX2 +ANYDETILE(DetileRow_Any_AVX2, DetileRow_AVX2, uint8_t, 1, 15) #endif #ifdef HAS_DETILEROW_16_NEON ANYDETILE(DetileRow_16_Any_NEON, DetileRow_16_NEON, uint16_t, 2, 15) #endif -#ifdef HAS_DETILEROW_16_SSE2 -ANYDETILE(DetileRow_16_Any_SSE2, DetileRow_16_SSE2, uint16_t, 2, 15) -#endif -#ifdef HAS_DETILEROW_16_AVX -ANYDETILE(DetileRow_16_Any_AVX, DetileRow_16_AVX, uint16_t, 2, 15) +#ifdef HAS_DETILEROW_16_AVX2 +ANYDETILE(DetileRow_16_Any_AVX2, DetileRow_16_AVX2, uint16_t, 2, 15) #endif // DetileSplitUVRow width is in bytes @@ -2934,17 +3003,17 @@ ANYDETILE(DetileRow_16_Any_AVX, DetileRow_16_AVX, uint16_t, 2, 15) #ifdef HAS_DETILESPLITUVROW_NEON ANYDETILESPLITUV(DetileSplitUVRow_Any_NEON, DetileSplitUVRow_NEON, 15) #endif -#ifdef HAS_DETILESPLITUVROW_SSSE3 -ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15) +#ifdef HAS_DETILESPLITUVROW_AVX2 +ANYDETILESPLITUV(DetileSplitUVRow_Any_AVX2, DetileSplitUVRow_AVX2, 15) #endif #define ANYDETILEMERGE(NAMEANY, ANY_SIMD, MASK) \ void NAMEANY(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, \ const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, \ uint8_t* dst_yuy2, int width) { \ - SIMD_ALIGNED(uint8_t vin[16 * 2]); \ + SIMD_ALIGNED(uint8_t vin[32 * 2]); \ static_assert(MASK + 1 <= sizeof(vin) / 2, "vin buffer too small"); \ - SIMD_ALIGNED(uint8_t vout[16 * 2]); \ + SIMD_ALIGNED(uint8_t vout[32 * 2]); \ static_assert(2 * (MASK + 1) <= sizeof(vout), "vout buffer too small"); \ MSAN_MEMSET(vin, 0, sizeof(vin)); /* for msan */ \ MSAN_MEMSET(vout, 0, sizeof(vout)); /* for msan */ \ @@ -2955,9 +3024,14 @@ ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15) n); \ } \ ptrdiff_t np = n; \ - memcpy(vin, src_y + (np / 16) * src_y_tile_stride, r); \ - memcpy(vin + 16, src_uv + (np / 16) * src_uv_tile_stride, r); \ - ANY_SIMD(vin, src_y_tile_stride, vin + 16, src_uv_tile_stride, vout, r); \ + int r0 = (r < 16) ? r : 16; \ + memcpy(vin, src_y + (np / 16) * src_y_tile_stride, r0); \ + memcpy(vin + 32, src_uv + (np / 16) * src_uv_tile_stride, r0); \ + if (r > 16) { \ + memcpy(vin + 16, src_y + (np / 16 + 1) * src_y_tile_stride, r - 16); \ + memcpy(vin + 48, src_uv + (np / 16 + 1) * src_uv_tile_stride, r - 16); \ + } \ + ANY_SIMD(vin, 16, vin + 32, 16, vout, MASK + 1); \ memcpy(dst_yuy2 + 2 * np, vout, 2 * r); \ } @@ -2965,10 +3039,12 @@ ANYDETILESPLITUV(DetileSplitUVRow_Any_SSSE3, DetileSplitUVRow_SSSE3, 15) ANYDETILEMERGE(DetileToYUY2_Any_NEON, DetileToYUY2_NEON, 15) #endif -#ifdef HAS_DETILETOYUY2_SSE2 -ANYDETILEMERGE(DetileToYUY2_Any_SSE2, DetileToYUY2_SSE2, 15) +#ifdef HAS_DETILETOYUY2_AVX2 +ANYDETILEMERGE(DetileToYUY2_Any_AVX2, DetileToYUY2_AVX2, 31) #endif +#undef ANYDETILEMERGE + #undef MSAN_MEMSET #ifdef __cplusplus diff --git a/source/row_common.cc b/source/row_common.cc index 94de60ab3..951d3c927 100644 --- a/source/row_common.cc +++ b/source/row_common.cc @@ -3643,20 +3643,20 @@ void ARGBAffineRow_C(const uint8_t* src_argb, } // Blend 2 rows into 1. -static void HalfRow_C(const uint8_t* src_uv, - ptrdiff_t src_uv_stride, - uint8_t* dst_uv, - int width) { +void HalfRow_C(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { int x; for (x = 0; x < width; ++x) { dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1; } } -static void HalfRow_16_C(const uint16_t* src_uv, - ptrdiff_t src_uv_stride, - uint16_t* dst_uv, - int width) { +void HalfRow_16_C(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { int x; for (x = 0; x < width; ++x) { dst_uv[x] = (src_uv[x] + src_uv[src_uv_stride + x] + 1) >> 1; @@ -4197,6 +4197,77 @@ void NV21ToRGB24Row_AVX2(const uint8_t* src_y, } #endif +#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTORGB565ROW_AVX512BW) +void I422ToRGB565Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_rgb565, + const struct YuvConstants* yuvconstants, + int width) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth); + ARGBToRGB565Row_AVX512BW(row, dst_rgb565, twidth); + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_rgb565 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTOARGB1555ROW_AVX2) +void I422ToARGB1555Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb1555, + const struct YuvConstants* yuvconstants, + int width) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth); + if (IS_ALIGNED(twidth, 8)) { + ARGBToARGB1555Row_AVX2(row, dst_argb1555, twidth); + } else { + ARGBToARGB1555Row_Any_AVX2(row, dst_argb1555, twidth); + } + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_argb1555 += twidth * 2; + width -= twidth; + } +} +#endif + +#if defined(HAS_I422TOARGBROW_AVX512BW) && defined(HAS_ARGBTOARGB4444ROW_AVX2) +void I422ToARGB4444Row_AVX512BW(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb4444, + const struct YuvConstants* yuvconstants, + int width) { + SIMD_ALIGNED(uint8_t row[MAXTWIDTH * 4]); + while (width > 0) { + int twidth = width > MAXTWIDTH ? MAXTWIDTH : width; + I422ToARGBRow_AVX512BW(src_y, src_u, src_v, row, yuvconstants, twidth); + if (IS_ALIGNED(twidth, 8)) { + ARGBToARGB4444Row_AVX2(row, dst_argb4444, twidth); + } else { + ARGBToARGB4444Row_Any_AVX2(row, dst_argb4444, twidth); + } + src_y += twidth; + src_u += twidth / 2; + src_v += twidth / 2; + dst_argb4444 += twidth * 2; + width -= twidth; + } +} +#endif + #if defined(HAS_I422TOARGBROW_AVX2) && defined(HAS_ARGBTORGB565ROW_AVX2) void I422ToRGB565Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, @@ -5056,6 +5127,61 @@ void ARGB4444ToUVMatrixRow_NEON(const uint8_t* src_argb4444, } #endif +#if defined(HAS_RGB565TOYROW_AVX2) && defined(HAS_RGB565TOYMATRIXROW_AVX2) +void RGB565ToYRow_AVX2(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { + RGB565ToYMatrixRow_AVX2(src_rgb565, dst_y, width, &kArgbI601Constants); +} +#endif + +#if defined(HAS_ARGB1555TOYROW_AVX2) && defined(HAS_ARGB1555TOYMATRIXROW_AVX2) +void ARGB1555ToYRow_AVX2(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width) { + ARGB1555ToYMatrixRow_AVX2(src_argb1555, dst_y, width, &kArgbI601Constants); +} +#endif + +#if defined(HAS_ARGB4444TOYROW_AVX2) && defined(HAS_ARGB4444TOYMATRIXROW_AVX2) +void ARGB4444ToYRow_AVX2(const uint8_t* src_argb4444, + uint8_t* dst_y, + int width) { + ARGB4444ToYMatrixRow_AVX2(src_argb4444, dst_y, width, &kArgbI601Constants); +} +#endif + +#if defined(HAS_RGB565TOUVROW_AVX2) && defined(HAS_RGB565TOUVMATRIXROW_AVX2) +void RGB565ToUVRow_AVX2(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + RGB565ToUVMatrixRow_AVX2(src_rgb565, src_stride_rgb565, dst_u, dst_v, width, + &kArgbI601Constants); +} +#endif + +#if defined(HAS_ARGB1555TOUVROW_AVX2) && defined(HAS_ARGB1555TOUVMATRIXROW_AVX2) +void ARGB1555ToUVRow_AVX2(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + ARGB1555ToUVMatrixRow_AVX2(src_argb1555, src_stride_argb1555, dst_u, dst_v, + width, &kArgbI601Constants); +} +#endif + +#if defined(HAS_ARGB4444TOUVROW_AVX2) && defined(HAS_ARGB4444TOUVMATRIXROW_AVX2) +void ARGB4444ToUVRow_AVX2(const uint8_t* src_argb4444, + int src_stride_argb4444, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + ARGB4444ToUVMatrixRow_AVX2(src_argb4444, src_stride_argb4444, dst_u, dst_v, + width, &kArgbI601Constants); +} +#endif + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/row_gcc.cc b/source/row_gcc.cc index 40f990dc0..876fdd274 100644 --- a/source/row_gcc.cc +++ b/source/row_gcc.cc @@ -32,30 +32,6 @@ extern "C" { // clang for windows needing to preserve xmm registers but not saving // them if declared as ymm or zmm. -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3) - -// Constants for ARGB - -// JPeg full range. -static const uvec8 kARGBToYJ = {29u, 150u, 77u, 0u, 29u, 150u, 77u, 0u, - 29u, 150u, 77u, 0u, 29u, 150u, 77u, 0u}; - -#endif // defined(HAS_ARGBTOYMATRIXROW_SSSE3) || defined(HAS_ARGBGRAYROW_SSSE3) - -#if defined(HAS_ARGBTOYMATRIXROW_SSSE3) || defined(HAS_I422TOARGBROW_SSSE3) -// Constants for BGRA - -// Constants for ABGR - -// Constants for RGBA. -// 126 (7e) - (-109..110) = 16..235 - -static const uvec16 kSub128 = {0x8080u, 0x8080u, 0x8080u, 0x8080u, - 0x8080u, 0x8080u, 0x8080u, 0x8080u}; - -#endif // defined(HAS_ARGBTOYMATRIXROW_SSSE3) || - // defined(HAS_I422TOARGBROW_SSSE3) - #if defined(HAS_RGB24TOARGBROW_SSSE3) || defined(HAS_RGB24TOARGBROW_AVX2) || \ defined(HAS_RGB24TOARGBROW_AVX512BW) || \ defined(HAS_RGBTOYMATRIXROW_AVX512BW) @@ -139,11 +115,13 @@ static const uint8_t kPermRAWToRGB24_3[64] = { 0x79, 0x78, 0x77, 0x7c, 0x7b, 0x7a, 0x7f, 0x7e, 0x7d}; #endif // defined(HAS_RAWTORGB24ROW_AVX512VBMI) -#if defined(HAS_RAWTORGBAROW_SSSE3) +#if defined(HAS_RAWTORGBAROW_AVX2) // Shuffle table for converting RAW to RGBA. -static const uvec8 kShuffleMaskRAWToRGBA = { - 128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u}; -#endif // defined(HAS_RAWTORGBAROW_SSSE3) +static const uvec8 kShuffleMaskRAWToRGBA_AVX2[2] = { + {128u, 2u, 1u, 0u, 128u, 5u, 4u, 3u, 128u, 8u, 7u, 6u, 128u, 11u, 10u, 9u}, + {128u, 6u, 5u, 4u, 128u, 9u, 8u, 7u, 128u, 12u, 11u, 10u, 128u, 15u, 14u, + 13u}}; +#endif // defined(HAS_RAWTORGBAROW_AVX2) #if defined(HAS_RAWTORGB24ROW_SSSE3) // Shuffle table for converting RAW to RGB24. First 8. @@ -576,46 +554,38 @@ void RGB24ToARGBRow_AVX512BW(const uint8_t* src_rgb24, } #endif // HAS_RAWTOARGBROW_AVX512BW -#ifdef HAS_RAWTORGBAROW_SSSE3 -// Same code as RAWToARGB with different shuffler and A in low bits -void RAWToRGBARow_SSSE3(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { +#ifdef HAS_RAWTORGBAROW_AVX2 +void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { asm volatile( - "pcmpeqb %%xmm5,%%xmm5 \n" // 0x000000ff - "psrld $24,%%xmm5 \n" - "movdqa %3,%%xmm4 \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsrld $24,%%ymm5,%%ymm5 \n" // 0x000000ff + "vbroadcasti128 %3,%%ymm3 \n" + "vmovdqu %3,%%ymm4 \n" LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "movdqu 0x20(%0),%%xmm3 \n" + "vmovdqu (%0),%%xmm0 \n" // first 12 bytes + "vinserti128 $1,12(%0),%%ymm0,%%ymm0 \n" // second 12 bytes + "vmovdqu 24(%0),%%xmm1 \n" // third 12 bytes + "vinserti128 $1,32(%0),%%ymm1,%%ymm1 \n" // fourth 12 bytes "lea 0x30(%0),%0 \n" - "movdqa %%xmm3,%%xmm2 \n" - "palignr $0x8,%%xmm1,%%xmm2 \n" - "pshufb %%xmm4,%%xmm2 \n" - "por %%xmm5,%%xmm2 \n" - "palignr $0xc,%%xmm0,%%xmm1 \n" - "pshufb %%xmm4,%%xmm0 \n" - "movdqu %%xmm2,0x20(%1) \n" - "por %%xmm5,%%xmm0 \n" - "pshufb %%xmm4,%%xmm1 \n" - "movdqu %%xmm0,(%1) \n" - "por %%xmm5,%%xmm1 \n" - "palignr $0x4,%%xmm3,%%xmm3 \n" - "pshufb %%xmm4,%%xmm3 \n" - "movdqu %%xmm1,0x10(%1) \n" - "por %%xmm5,%%xmm3 \n" - "movdqu %%xmm3,0x30(%1) \n" + "vpshufb %%ymm3,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm4,%%ymm1,%%ymm1 \n" + "vpor %%ymm5,%%ymm0,%%ymm0 \n" + "vpor %%ymm5,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm0,(%1) \n" + "vmovdqu %%ymm1,0x20(%1) \n" "lea 0x40(%1),%1 \n" "sub $0x10,%2 \n" "jg 1b \n" - : "+r"(src_raw), // %0 - "+r"(dst_rgba), // %1 - "+r"(width) // %2 - : "m"(kShuffleMaskRAWToRGBA) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); + "vzeroupper \n" + : "+r"(src_raw), // %0 + "+r"(dst_rgba), // %1 + "+r"(width) // %2 + : "m"(kShuffleMaskRAWToRGBA_AVX2[0]) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm3", "xmm4", "xmm5"); } -#endif // HAS_RAWTORGBAROW_SSSE3 +#endif // HAS_RAWTORGBAROW_AVX2 #ifdef HAS_RAWTORGB24ROW_SSSE3 void RAWToRGB24Row_SSSE3(const uint8_t* src_raw, @@ -908,6 +878,97 @@ void ARGB1555ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGB1555TOARGBROW_AVX2 +#ifdef HAS_ARGB1555TOARGBROW_AVX512BW +void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + uintptr_t temp; + asm volatile( + "mov $0x01080108,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm5 \n" + "mov $0x42004200,%k[temp] \n" + "vpbroadcastd %k[temp],%%zmm6 \n" + "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n" + "vpsllw $0xb,%%zmm3,%%zmm3 \n" + "vpsrlw $0x6,%%zmm3,%%zmm4 \n" + "vpternlogd $0xff,%%zmm7,%%zmm7,%%zmm7 \n" + "vpsllw $0x8,%%zmm7,%%zmm7 \n" + "sub $0x20,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu16 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsllw $0x1,%%zmm0,%%zmm1 \n" + "vpsllw $0xb,%%zmm0,%%zmm2 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpmulhuw %%zmm5,%%zmm2,%%zmm2 \n" + "vpmulhuw %%zmm5,%%zmm1,%%zmm1 \n" + "vpsllw $0x8,%%zmm1,%%zmm1 \n" + "vpord %%zmm2,%%zmm1,%%zmm1 \n" + "vpsraw $0x8,%%zmm0,%%zmm2 \n" + "vpandd %%zmm4,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n" + "vpandd %%zmm7,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vpunpcklbw %%zmm0,%%zmm1,%%zmm2 \n" + "vpunpckhbw %%zmm0,%%zmm1,%%zmm1 \n" + "vshufi64x2 $0x44,%%zmm1,%%zmm2,%%zmm0 \n" + "vshufi64x2 $0xee,%%zmm1,%%zmm2,%%zmm1 \n" + "vshufi64x2 $0xd8,%%zmm0,%%zmm0,%%zmm0 \n" + "vshufi64x2 $0xd8,%%zmm1,%%zmm1,%%zmm1 \n" + "vmovdqu32 %%zmm0,(%1) \n" + "vmovdqu32 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpsllw $0x1,%%zmm0,%%zmm1 \n" + "vpsllw $0xb,%%zmm0,%%zmm2 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpmulhuw %%zmm5,%%zmm2,%%zmm2 \n" + "vpmulhuw %%zmm5,%%zmm1,%%zmm1 \n" + "vpsllw $0x8,%%zmm1,%%zmm1 \n" + "vpord %%zmm2,%%zmm1,%%zmm1 \n" + "vpsraw $0x8,%%zmm0,%%zmm2 \n" + "vpandd %%zmm4,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm6,%%zmm0,%%zmm0 \n" + "vpandd %%zmm7,%%zmm2,%%zmm2 \n" + "vpord %%zmm2,%%zmm0,%%zmm0 \n" + "vpunpcklbw %%zmm0,%%zmm1,%%zmm2 \n" + "vpunpckhbw %%zmm0,%%zmm1,%%zmm1 \n" + "vshufi64x2 $0x44,%%zmm1,%%zmm2,%%zmm0 \n" + "vshufi64x2 $0xee,%%zmm1,%%zmm2,%%zmm1 \n" + "vshufi64x2 $0xd8,%%zmm0,%%zmm0,%%zmm0 \n" + "vshufi64x2 $0xd8,%%zmm1,%%zmm1,%%zmm1 \n" + "cmp $0x10,%2 \n" + "jle 3f \n" + "vmovdqu32 %%zmm0,(%1) \n" + "kshiftrd $0x10,%%k1,%%k1 \n" + "vmovdqu32 %%zmm1,0x40(%1)%{%%k1%} \n" + "jmp 99f \n" + + "3: \n" + "vmovdqu32 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "k1"); +} +#endif // HAS_ARGB1555TOARGBROW_AVX512BW + #ifdef HAS_ARGB4444TOARGBROW_AVX2 void ARGB4444ToARGBRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { asm volatile( @@ -1273,6 +1334,72 @@ void ARGBToRGB565DitherRow_AVX2(const uint8_t* src, } #endif // HAS_ARGBTORGB565DITHERROW_AVX2 +#ifdef HAS_ARGBTORGB565DITHERROW_AVX512BW +void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src, + uint8_t* dst, + uint32_t dither4, + int width) { + uintptr_t temp; + asm volatile( + "vmovd %[dither4],%%xmm6 \n" + "vpunpcklbw %%xmm6,%%xmm6,%%xmm6 \n" + "vpunpcklwd %%xmm6,%%xmm6,%%xmm6 \n" + "vshufi32x4 $0x00,%%zmm6,%%zmm6,%%zmm6 \n" + "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n" + "vpsrld $0x1b,%%zmm3,%%zmm3 \n" + "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n" + "vpsrld $0x1a,%%zmm4,%%zmm4 \n" + "vpslld $0x5,%%zmm4,%%zmm4 \n" + "vpslld $0xb,%%zmm3,%%zmm5 \n" + "sub $0x10,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpaddusb %%zmm6,%%zmm0,%%zmm0 \n" + "vpsrld $0x3,%%zmm0,%%zmm1 \n" + "vpsrld $0x5,%%zmm0,%%zmm2 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpandd %%zmm4,%%zmm2,%%zmm2 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n" + "vpmovdw %%zmm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpaddusb %%zmm6,%%zmm0,%%zmm0 \n" + "vpsrld $0x3,%%zmm0,%%zmm1 \n" + "vpsrld $0x5,%%zmm0,%%zmm2 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpandd %%zmm4,%%zmm2,%%zmm2 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n" + "vpmovdw %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [dither4] "r"(dither4) // %[dither4] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "k1"); +} +#endif // HAS_ARGBTORGB565DITHERROW_AVX512BW + #ifdef HAS_ARGBTORGB565ROW_AVX2 void ARGBToRGB565Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { asm volatile( @@ -1310,6 +1437,62 @@ void ARGBToRGB565Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBTORGB565ROW_AVX2 +#ifdef HAS_ARGBTORGB565ROW_AVX512BW +void ARGBToRGB565Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + uintptr_t temp; + asm volatile( + "vpternlogd $0xff,%%zmm3,%%zmm3,%%zmm3 \n" + "vpsrld $0x1b,%%zmm3,%%zmm3 \n" + "vpternlogd $0xff,%%zmm4,%%zmm4,%%zmm4 \n" + "vpsrld $0x1a,%%zmm4,%%zmm4 \n" + "vpslld $0x5,%%zmm4,%%zmm4 \n" + "vpslld $0xb,%%zmm3,%%zmm5 \n" + "sub $0x10,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x3,%%zmm0,%%zmm1 \n" + "vpsrld $0x5,%%zmm0,%%zmm2 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpandd %%zmm4,%%zmm2,%%zmm2 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n" + "vpmovdw %%zmm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpsrld $0x3,%%zmm0,%%zmm1 \n" + "vpsrld $0x5,%%zmm0,%%zmm2 \n" + "vpsrld $0x8,%%zmm0,%%zmm0 \n" + "vpandd %%zmm3,%%zmm1,%%zmm1 \n" + "vpandd %%zmm4,%%zmm2,%%zmm2 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpternlogd $0xfe,%%zmm2,%%zmm1,%%zmm0 \n" + "vpmovdw %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1"); +} +#endif // HAS_ARGBTORGB565ROW_AVX512BW + #ifdef HAS_ARGBTOARGB1555ROW_AVX2 void ARGBToARGB1555Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { asm volatile( @@ -3365,6 +3548,8 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "psubsw %%xmm1,%%xmm4 \n" \ "movdqa %%xmm4,%%xmm1 \n" +#define YUVTORGB16_AR30(yuvconstants) YUVTORGB16(yuvconstants) + #define YUVTORGB_REGS "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", #else @@ -3391,6 +3576,29 @@ void ARGBToUVMatrixRow_AVX512VNNIVBMI(const uint8_t* src_argb, "psubsw %%xmm1,%%xmm4 \n" \ "movdqa %%xmm4,%%xmm1 \n" +#define YUVTORGB16_AR30(yuvconstants) \ + "pcmpeqb %%xmm0,%%xmm0 \n" \ + "pabsb %%xmm0,%%xmm0 \n" \ + "psllw $7,%%xmm0 \n" \ + "psubb %%xmm0,%%xmm3 \n" \ + "pmulhuw 96(%[yuvconstants]),%%xmm4 \n" \ + "movdqa (%[yuvconstants]),%%xmm0 \n" \ + "movdqa 32(%[yuvconstants]),%%xmm1 \n" \ + "movdqa 64(%[yuvconstants]),%%xmm2 \n" \ + "pmaddubsw %%xmm3,%%xmm0 \n" \ + "pmaddubsw %%xmm3,%%xmm1 \n" \ + "pmaddubsw %%xmm3,%%xmm2 \n" \ + "movdqa 128(%[yuvconstants]),%%xmm3 \n" \ + "paddw %%xmm3,%%xmm4 \n" \ + "pcmpeqb %%xmm3,%%xmm3 \n" \ + "psrlw $14,%%xmm3 \n" \ + "psllw $3,%%xmm3 \n" \ + "psubw %%xmm3,%%xmm4 \n" \ + "paddsw %%xmm4,%%xmm0 \n" \ + "paddsw %%xmm4,%%xmm2 \n" \ + "psubsw %%xmm1,%%xmm4 \n" \ + "movdqa %%xmm4,%%xmm1 \n" + #define YUVTORGB_REGS #endif // defined(__x86_64__) @@ -3501,6 +3709,43 @@ void OMITFP I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "movd %[temp],%%xmm7 \n" + "movd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "movd %%xmm7,%[temp] \n" + "movq (%[u_buf]),%%xmm3 \n" + "movq 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x8(%[u_buf]),%[u_buf] \n" + "punpcklbw %%xmm1,%%xmm3 \n" + "movq (%[y_buf]),%%xmm4 \n" + "punpcklbw %%xmm4,%%xmm4 \n" + "lea 0x8(%[y_buf]),%[y_buf] \n" + "movd %%xmm6,%[temp] \n" + "movq (%[temp]),%%xmm5 \n" + "lea 0x8(%[temp]),%[temp] \n" + "movd %[temp],%%xmm6 \n" + YUVTORGB(yuvconstants) + STOREARGB + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -3513,14 +3758,11 @@ void OMITFP I444AlphaToARGBRow_SSSE3(const uint8_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // HAS_I444ALPHATOARGBROW_SSSE3 @@ -3625,6 +3867,7 @@ void OMITFP I422ToARGBRow_SSSE3(const uint8_t* y_buf, ); } +#ifdef HAS_I422TOAR30ROW_SSSE3 void OMITFP I422ToAR30Row_SSSE3(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -3644,20 +3887,25 @@ void OMITFP I422ToAR30Row_SSSE3(const uint8_t* y_buf, LABELALIGN "1: \n" READYUV422 - YUVTORGB16(yuvconstants) + YUVTORGB16_AR30(yuvconstants) STOREAR30 - "sub $0x8,%[width] \n" + "subl $0x8,%[width] \n" "jg 1b \n" : [y_buf]"+r"(y_buf), // %[y_buf] [u_buf]"+r"(u_buf), // %[u_buf] [v_buf]"+r"(v_buf), // %[v_buf] [dst_ar30]"+r"(dst_ar30), // %[dst_ar30] +#if defined(__i386__) + [width]"+m"(width) // %[width] +#else [width]"+rm"(width) // %[width] +#endif : [yuvconstants]"r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" ); } +#endif // HAS_I422TOAR30ROW_SSSE3 // 10 bit YUV to ARGB void OMITFP I210ToARGBRow_SSSE3(const uint16_t* y_buf, @@ -3827,6 +4075,51 @@ void OMITFP I210AlphaToARGBRow_SSSE3(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "movd %[temp],%%xmm7 \n" + "movd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "movd %%xmm7,%[temp] \n" + "movq (%[u_buf]),%%xmm3 \n" + "movq 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x8(%[u_buf]),%[u_buf] \n" + "punpcklwd %%xmm1,%%xmm3 \n" + "psraw $2,%%xmm3 \n" + "packuswb %%xmm3,%%xmm3 \n" + "punpcklwd %%xmm3,%%xmm3 \n" + "movdqu (%[y_buf]),%%xmm4 \n" + "movdqa %%xmm4,%%xmm2 \n" + "psllw $6,%%xmm4 \n" + "psrlw $4,%%xmm2 \n" + "paddw %%xmm2,%%xmm4 \n" + "lea 0x10(%[y_buf]),%[y_buf] \n" + "movd %%xmm6,%[temp] \n" + "movdqu (%[temp]),%%xmm5 \n" + "psraw $2,%%xmm5 \n" + "packuswb %%xmm5,%%xmm5 \n" + "lea 0x10(%[temp]),%[temp] \n" + "movd %[temp],%%xmm6 \n" + YUVTORGB(yuvconstants) + STOREARGB + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -3839,14 +4132,11 @@ void OMITFP I210AlphaToARGBRow_SSSE3(const uint16_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // HAS_I210ALPHATOARGBROW_SSSE3 @@ -3859,6 +4149,53 @@ void OMITFP I410AlphaToARGBRow_SSSE3(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "movd %[temp],%%xmm7 \n" + "movd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "movd %%xmm7,%[temp] \n" + "movdqu (%[u_buf]),%%xmm3 \n" + "movdqu 0x00(%[u_buf],%[temp],1),%%xmm2 \n" + "lea 0x10(%[u_buf]),%[u_buf] \n" + "psraw $2,%%xmm3 \n" + "psraw $2,%%xmm2 \n" + "movdqa %%xmm3,%%xmm1 \n" + "punpcklwd %%xmm2,%%xmm3 \n" + "punpckhwd %%xmm2,%%xmm1 \n" + "packuswb %%xmm1,%%xmm3 \n" + "movdqu (%[y_buf]),%%xmm4 \n" + "movdqa %%xmm4,%%xmm2 \n" + "psllw $6,%%xmm4 \n" + "psrlw $4,%%xmm2 \n" + "paddw %%xmm2,%%xmm4 \n" + "lea 0x10(%[y_buf]),%[y_buf] \n" + "movd %%xmm6,%[temp] \n" + "movdqu (%[temp]),%%xmm5 \n" + "psraw $2,%%xmm5 \n" + "packuswb %%xmm5,%%xmm5 \n" + "lea 0x10(%[temp]),%[temp] \n" + "movd %[temp],%%xmm6 \n" + YUVTORGB(yuvconstants) + STOREARGB + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -3871,14 +4208,11 @@ void OMITFP I410AlphaToARGBRow_SSSE3(const uint16_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // HAS_I410ALPHATOARGBROW_SSSE3 @@ -3925,6 +4259,44 @@ void OMITFP I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "movd %[temp],%%xmm7 \n" + "movd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "movd %%xmm7,%[temp] \n" + "movd (%[u_buf]),%%xmm3 \n" + "movd 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x4(%[u_buf]),%[u_buf] \n" + "punpcklbw %%xmm1,%%xmm3 \n" + "punpcklwd %%xmm3,%%xmm3 \n" + "movq (%[y_buf]),%%xmm4 \n" + "punpcklbw %%xmm4,%%xmm4 \n" + "lea 0x8(%[y_buf]),%[y_buf] \n" + "movd %%xmm6,%[temp] \n" + "movq (%[temp]),%%xmm5 \n" + "lea 0x8(%[temp]),%[temp] \n" + "movd %[temp],%%xmm6 \n" + YUVTORGB(yuvconstants) + STOREARGB + "subl $0x8,%[width] \n" + "jg 1b \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -3937,14 +4309,11 @@ void OMITFP I422AlphaToARGBRow_SSSE3(const uint8_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // HAS_I422ALPHATOARGBROW_SSSE3 @@ -4466,6 +4835,8 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \ "vpaddsw %%ymm4,%%ymm2,%%ymm2 \n" +#define YUVTORGB16_AR30_AVX2(yuvconstants) YUVTORGB16_AVX2(yuvconstants) + #define YUVTORGB16_AVX512BW(yuvconstants) \ "vpsubb %%zmm13,%%zmm3,%%zmm3 \n" \ "vpmulhuw %%zmm11,%%zmm4,%%zmm4 \n" \ @@ -4504,6 +4875,28 @@ void OMITFP I422ToRGBARow_SSSE3(const uint8_t* y_buf, "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \ "vpaddsw %%ymm4,%%ymm2,%%ymm2 \n" +#define YUVTORGB16_AR30_AVX2(yuvconstants) \ + "vpcmpeqb %%ymm0,%%ymm0,%%ymm0 \n" \ + "vpabsb %%ymm0,%%ymm0 \n" \ + "vpsllw $7,%%ymm0,%%ymm0 \n" \ + "vpsubb %%ymm0,%%ymm3,%%ymm3 \n" \ + "vpmulhuw 96(%[yuvconstants]),%%ymm4,%%ymm4 \n" \ + "vmovdqa (%[yuvconstants]),%%ymm0 \n" \ + "vmovdqa 32(%[yuvconstants]),%%ymm1 \n" \ + "vmovdqa 64(%[yuvconstants]),%%ymm2 \n" \ + "vpmaddubsw %%ymm3,%%ymm0,%%ymm0 \n" \ + "vpmaddubsw %%ymm3,%%ymm1,%%ymm1 \n" \ + "vpmaddubsw %%ymm3,%%ymm2,%%ymm2 \n" \ + "vmovdqa 128(%[yuvconstants]),%%ymm3 \n" \ + "vpaddw %%ymm4,%%ymm3,%%ymm4 \n" \ + "vpcmpeqb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpsrlw $14,%%ymm3,%%ymm3 \n" \ + "vpsllw $3,%%ymm3,%%ymm3 \n" \ + "vpsubw %%ymm3,%%ymm4,%%ymm4 \n" \ + "vpaddsw %%ymm4,%%ymm0,%%ymm0 \n" \ + "vpsubsw %%ymm1,%%ymm4,%%ymm1 \n" \ + "vpaddsw %%ymm4,%%ymm2,%%ymm2 \n" + #define YUVTORGB_REGS_AVX2 #endif // defined(__x86_64__) @@ -4827,28 +5220,46 @@ void OMITFP I422ToRGB24Row_AVX2(const uint8_t* y_buf, } #endif // defined(HAS_I422TORGB24ROW_AVX2) -#if defined(HAS_I422TOARGBROW_AVX512BW) || \ - defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ - defined(HAS_I422TORGB24ROW_AVX512BW) || \ - defined(HAS_I422TOAR30ROW_AVX512BW) +#if defined(HAS_I422TOARGBROW_AVX512BW) || \ + defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I422TORGB24ROW_AVX512BW) || \ + defined(HAS_I422TOAR30ROW_AVX512BW) || \ + defined(HAS_I444TOARGBROW_AVX512BW) || \ + defined(HAS_I444ALPHATOARGBROW_AVX512BW) || \ + defined(HAS_I444TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I422TORGBAROW_AVX512BW) || \ + defined(HAS_YUY2TOARGBROW_AVX512BW) || \ + defined(HAS_UYVYTOARGBROW_AVX512BW) || \ + defined(HAS_I210TOARGBROW_AVX512BW) || \ + defined(HAS_I210TOAR30ROW_AVX512BW) || \ + defined(HAS_I210ALPHATOARGBROW_AVX512BW) || \ + defined(HAS_I212TOARGBROW_AVX512BW) || \ + defined(HAS_I212TOAR30ROW_AVX512BW) || \ + defined(HAS_I410TOARGBROW_AVX512BW) || \ + defined(HAS_I410TOAR30ROW_AVX512BW) || \ + defined(HAS_I410ALPHATOARGBROW_AVX512BW) || \ + defined(HAS_P210TOARGBROW_AVX512BW) || \ + defined(HAS_P210TOAR30ROW_AVX512BW) || \ + defined(HAS_P410TOARGBROW_AVX512BW) || defined(HAS_P410TOAR30ROW_AVX512BW) static const uint16_t kSplitQuadWords[32] = { 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15}; static const uint64_t kSplitDoubleQuadWords[8] = {0, 1, 4, 4, 2, 3, 4, 4}; static const uint64_t kUnpermuteAVX512[8] = {0, 4, 1, 5, 2, 6, 3, 7}; -#endif // defined(HAS_I422TOARGBROW_AVX512BW) || - // defined(HAS_I422TORGB24ROW_AVX512VBMI) || - // defined(HAS_I422TORGB24ROW_AVX512BW) || - // defined(HAS_I422TOAR30ROW_AVX512BW) +#endif -#if defined(HAS_I422TOAR30ROW_AVX512BW) +#if defined(HAS_I422TOAR30ROW_AVX512BW) || \ + defined(HAS_I210TOAR30ROW_AVX512BW) || \ + defined(HAS_I212TOAR30ROW_AVX512BW) || \ + defined(HAS_I410TOAR30ROW_AVX512BW) || \ + defined(HAS_P210TOAR30ROW_AVX512BW) || defined(HAS_P410TOAR30ROW_AVX512BW) static const uint16_t kPermAR30_0[32] = { 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47}; static const uint16_t kPermAR30_1[32] = { 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63}; -#endif // defined(HAS_I422TOAR30ROW_AVX512BW) +#endif #if defined(HAS_I422TOARGBROW_AVX512BW) // 32 pixels @@ -4903,7 +5314,8 @@ void OMITFP I422ToARGBRow_AVX512BW(const uint8_t* y_buf, } #endif // defined(HAS_I422TOARGBROW_AVX512BW) -#if defined(HAS_I422TORGB24ROW_AVX512VBMI) +#if defined(HAS_I422TORGB24ROW_AVX512VBMI) || \ + defined(HAS_I444TORGB24ROW_AVX512VBMI) static const uint8_t kMaskBG[64] = { 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, @@ -4925,7 +5337,10 @@ static const uint8_t kMaskDST1[64] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +#endif // defined(HAS_I422TORGB24ROW_AVX512VBMI) || + // defined(HAS_I444TORGB24ROW_AVX512VBMI) +#if defined(HAS_I422TORGB24ROW_AVX512VBMI) // 32 pixels // 16 UV values upsampled to 32 UV, mixed with 32 Y producing 32 RGB24 (96 // bytes). @@ -5131,9 +5546,9 @@ void OMITFP I422ToAR30Row_AVX2(const uint8_t* y_buf, LABELALIGN "1: \n" READYUV422_AVX2 - YUVTORGB16_AVX2(yuvconstants) + YUVTORGB16_AR30_AVX2(yuvconstants) STOREAR30_AVX2 - "sub $0x10,%[width] \n" + "subl $0x10,%[width] \n" "jg 1b \n" "vzeroupper \n" @@ -5141,7 +5556,11 @@ void OMITFP I422ToAR30Row_AVX2(const uint8_t* y_buf, [u_buf]"+r"(u_buf), // %[u_buf] [v_buf]"+r"(v_buf), // %[v_buf] [dst_ar30]"+r"(dst_ar30), // %[dst_ar30] +#if defined(__i386__) + [width]"+m"(width) // %[width] +#else [width]"+rm"(width) // %[width] +#endif : [yuvconstants]"r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7" @@ -5342,6 +5761,53 @@ void OMITFP I210AlphaToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "vmovd %[temp],%%xmm7 \n" + "vmovd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "vmovd %%xmm7,%[temp] \n" + "vmovdqu (%[u_buf]),%%xmm3 \n" + "vmovdqu 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x10(%[u_buf]),%[u_buf] \n" + "vpermq $0xd8,%%ymm3,%%ymm3 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpunpcklwd %%ymm1,%%ymm3,%%ymm3 \n" + "vpsraw $2,%%ymm3,%%ymm3 \n" + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" + "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" + "vmovdqu (%[y_buf]),%%ymm4 \n" + "vpsllw $6,%%ymm4,%%ymm2 \n" + "vpsrlw $4,%%ymm4,%%ymm4 \n" + "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" + "lea 0x20(%[y_buf]),%[y_buf] \n" + "vmovd %%xmm6,%[temp] \n" + "vmovdqu (%[temp]),%%ymm5 \n" + "vpsraw $2,%%ymm5,%%ymm5 \n" + "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n" + "lea 0x20(%[temp]),%[temp] \n" + "vmovd %[temp],%%xmm6 \n" + YUVTORGB_AVX2(yuvconstants) + STOREARGB_AVX2 + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP_AVX2( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -5356,14 +5822,11 @@ void OMITFP I210AlphaToARGBRow_AVX2(const uint16_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // defined(HAS_I210ALPHATOARGBROW_AVX2) @@ -5377,6 +5840,52 @@ void OMITFP I410AlphaToARGBRow_AVX2(const uint16_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "vmovd %[temp],%%xmm7 \n" + "vmovd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "vmovd %%xmm7,%[temp] \n" + "vmovdqu (%[u_buf]),%%ymm3 \n" + "vmovdqu 0x00(%[u_buf],%[temp],1),%%ymm2 \n" + "lea 0x20(%[u_buf]),%[u_buf] \n" + "vpsraw $2,%%ymm3,%%ymm3 \n" + "vpsraw $2,%%ymm2,%%ymm2 \n" + "vpunpckhwd %%ymm2,%%ymm3,%%ymm1 \n" + "vpunpcklwd %%ymm2,%%ymm3,%%ymm3 \n" + "vpackuswb %%ymm1,%%ymm3,%%ymm3 \n" + "vmovdqu (%[y_buf]),%%ymm4 \n" + "vpsllw $6,%%ymm4,%%ymm2 \n" + "vpsrlw $4,%%ymm4,%%ymm4 \n" + "vpaddw %%ymm2,%%ymm4,%%ymm4 \n" + "lea 0x20(%[y_buf]),%[y_buf] \n" + "vmovd %%xmm6,%[temp] \n" + "vmovdqu (%[temp]),%%ymm5 \n" + "vpsraw $2,%%ymm5,%%ymm5 \n" + "vpackuswb %%ymm5,%%ymm5,%%ymm5 \n" + "lea 0x20(%[temp]),%[temp] \n" + "vmovd %[temp],%%xmm6 \n" + YUVTORGB_AVX2(yuvconstants) + STOREARGB_AVX2 + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP_AVX2( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -5391,14 +5900,11 @@ void OMITFP I410AlphaToARGBRow_AVX2(const uint16_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // defined(HAS_I410ALPHATOARGBROW_AVX2) @@ -5451,6 +5957,48 @@ void OMITFP I444AlphaToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "vmovd %[temp],%%xmm7 \n" + "vmovd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "vmovd %%xmm7,%[temp] \n" + "vmovdqu (%[u_buf]),%%xmm3 \n" + "vmovdqu 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x10(%[u_buf]),%[u_buf] \n" + "vpermq $0xd8,%%ymm3,%%ymm3 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" + "vmovdqu (%[y_buf]),%%xmm4 \n" + "vpermq $0xd8,%%ymm4,%%ymm4 \n" + "vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" + "lea 0x10(%[y_buf]),%[y_buf] \n" + "vmovd %%xmm6,%[temp] \n" + "vmovdqu (%[temp]),%%xmm5 \n" + "vpermq $0xd8,%%ymm5,%%ymm5 \n" + "lea 0x10(%[temp]),%[temp] \n" + "vmovd %[temp],%%xmm6 \n" + YUVTORGB_AVX2(yuvconstants) + STOREARGB_AVX2 + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP_AVX2( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -5464,14 +6012,11 @@ void OMITFP I444AlphaToARGBRow_AVX2(const uint8_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // defined(HAS_I444ALPHATOARGBROW_AVX2) @@ -5485,6 +6030,48 @@ void OMITFP I422AlphaToARGBRow_AVX2(const uint8_t* y_buf, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { +#if defined(__i386__) + uintptr_t temp; + asm volatile( + "mov %[v_buf],%[temp] \n" + "sub %[u_buf],%[temp] \n" + "vmovd %[temp],%%xmm7 \n" + "vmovd %[a_buf],%%xmm6 \n" + + LABELALIGN + "1: \n" + "vmovd %%xmm7,%[temp] \n" + "vmovq (%[u_buf]),%%xmm3 \n" + "vmovq 0x00(%[u_buf],%[temp],1),%%xmm1 \n" + "lea 0x8(%[u_buf]),%[u_buf] \n" + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" + "vpermq $0xd8,%%ymm3,%%ymm3 \n" + "vpunpcklwd %%ymm3,%%ymm3,%%ymm3 \n" + "vmovdqu (%[y_buf]),%%xmm4 \n" + "vpermq $0xd8,%%ymm4,%%ymm4 \n" + "vpunpcklbw %%ymm4,%%ymm4,%%ymm4 \n" + "lea 0x10(%[y_buf]),%[y_buf] \n" + "vmovd %%xmm6,%[temp] \n" + "vmovdqu (%[temp]),%%xmm5 \n" + "vpermq $0xd8,%%ymm5,%%ymm5 \n" + "lea 0x10(%[temp]),%[temp] \n" + "vmovd %[temp],%%xmm6 \n" + YUVTORGB_AVX2(yuvconstants) + STOREARGB_AVX2 + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [dst_argb] "+r"(dst_argb), + [temp] "=&r"(temp), + [width] "+m"(width) + : [v_buf] "m"(v_buf), + [a_buf] "m"(a_buf), + [yuvconstants] "r"(yuvconstants) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +#else asm volatile(YUVTORGB_SETUP_AVX2( yuvconstants) "sub %[u_buf],%[v_buf] \n" @@ -5498,14 +6085,11 @@ void OMITFP I422AlphaToARGBRow_AVX2(const uint8_t* y_buf, [v_buf] "+r"(v_buf), // %[v_buf] [a_buf] "+r"(a_buf), // %[a_buf] [dst_argb] "+r"(dst_argb), // %[dst_argb] -#if defined(__i386__) - [width] "+m"(width) // %[width] -#else - [width] "+rm"(width) // %[width] -#endif + [width] "+rm"(width) // %[width] : [yuvconstants] "r"(yuvconstants) // %[yuvconstants] : "memory", "cc", YUVTORGB_REGS_AVX2 "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif } #endif // defined(HAS_I422ALPHATOARGBROW_AVX2) @@ -5896,26 +6480,69 @@ void MirrorRow_SSSE3(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_MIRRORROW_SSSE3 +#if defined(HAS_MIRRORROW_AVX512BW) || \ + defined(HAS_MIRRORSPLITUVROW_AVX512BW) || \ + defined(HAS_MIRRORUVROW_AVX512BW) +static const uint16_t kMirrorUVPermute_AVX512[32] = { + 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u, + 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u, + 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; +#endif // defined(HAS_MIRRORROW_AVX512BW) || + // defined(HAS_MIRRORSPLITUVROW_AVX512BW) || + // defined(HAS_MIRRORUVROW_AVX512BW) + #ifdef HAS_MIRRORROW_AVX512BW void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); - asm volatile("vbroadcasti32x4 %3,%%zmm5 \n" + uintptr_t temp; + asm volatile( + "vbroadcasti32x4 %[shuf],%%zmm5 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" - LABELALIGN - "1: \n" - "vmovdqu8 -0x40(%0,%2,1),%%zmm0 \n" - "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" - "vshufi64x2 $0x1b,%%zmm0,%%zmm0,%%zmm0 \n" - "vmovdqu8 %%zmm0,(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x40,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(temp_width) // %2 - : "m"(kShuffleMirror) // %3 - : "memory", "cc", "xmm0", "xmm5"); + LABELALIGN + "1: \n" + "vmovdqu8 (%[src],%[width],1),%%zmm0 \n" + "vpshufb %%zmm5,%%zmm0,%%zmm0 \n" + "vshufi64x2 $0x1b,%%zmm0,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "je 99f \n" + "mov $-1,%[temp] \n" + "bzhi %[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "vmovdqu8 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpsrlw $0x8,%%zmm0,%%zmm1 \n" + "vpsllw $0x8,%%zmm0,%%zmm0 \n" + "vporq %%zmm1,%%zmm0,%%zmm0 \n" + "mov $0x40,%k[temp] \n" + "sub %k[width],%k[temp] \n" + "shr $0x1,%k[temp] \n" + "vpbroadcastw %k[temp],%%zmm1 \n" + "vmovdqu16 %[rev],%%zmm2 \n" + "vpsubw %%zmm1,%%zmm2,%%zmm1 \n" + "vpermw %%zmm0,%%zmm1,%%zmm0 \n" + "test $0x1,%b[width] \n" + "je 3f \n" + "valignq $0x2,%%zmm0,%%zmm0,%%zmm1 \n" + "vpalignr $0x1,%%zmm0,%%zmm1,%%zmm0 \n" + "3: \n" + "vmovdqu8 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst), + [width] "+r"(temp_width), + [temp] "=&r"(temp) + : [src] "r"(src), + [shuf] "m"(kShuffleMirror), + [rev] "m"(kMirrorUVPermute_AVX512) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm5", "k1"); } #endif // HAS_MIRRORROW_AVX512BW @@ -5949,14 +6576,6 @@ static const uvec8 kShuffleMirrorSplitUV = {14u, 12u, 10u, 8u, 6u, 4u, 2u, 0u, #endif // defined(HAS_MIRRORSPLITUVROW_AVX2) || // defined(HAS_MIRRORSPLITUVROW_AVX512BW) -#if defined(HAS_MIRRORSPLITUVROW_AVX512BW) || defined(HAS_MIRRORUVROW_AVX512BW) -static const uint16_t kMirrorUVPermute_AVX512[32] = { - 31u, 30u, 29u, 28u, 27u, 26u, 25u, 24u, 23u, 22u, 21u, - 20u, 19u, 18u, 17u, 16u, 15u, 14u, 13u, 12u, 11u, 10u, - 9u, 8u, 7u, 6u, 5u, 4u, 3u, 2u, 1u, 0u}; -#endif // defined(HAS_MIRRORSPLITUVROW_AVX512BW) || - // defined(HAS_MIRRORUVROW_AVX512BW) - #ifdef HAS_MIRRORSPLITUVROW_AVX512BW static const uint64_t kMirrorSplitUVPermute[8] = {6, 4, 2, 0, 7, 5, 3, 1}; @@ -6100,21 +6719,42 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { #ifdef HAS_MIRRORUVROW_AVX512BW void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { ptrdiff_t temp_width = (ptrdiff_t)(width); - asm volatile("vmovdqu16 %3,%%zmm5 \n" + uintptr_t temp; + asm volatile( + "vmovdqu16 %[rev],%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" - LABELALIGN - "1: \n" - "vpermw -0x40(%0,%2,2),%%zmm5,%%zmm0 \n" - "vmovdqu16 %%zmm0,(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x20,%2 \n" - "jg 1b \n" - "vzeroupper \n" - : "+r"(src_uv), // %0 - "+r"(dst_uv), // %1 - "+r"(temp_width) // %2 - : "m"(kMirrorUVPermute_AVX512) // %3 - : "memory", "cc", "xmm0", "xmm5"); + LABELALIGN + "1: \n" + "vpermw (%[src_uv],%[width],2),%%zmm5,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst_uv]) \n" + "lea 0x40(%[dst_uv]),%[dst_uv] \n" + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + "mov $-1,%k[temp] \n" + "bzhi %k[width],%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%[src_uv]),%%zmm0%{%%k1%}%{z%}\n" + "mov $0x20,%k[temp] \n" + "sub %k[width],%k[temp] \n" + "vpbroadcastw %k[temp],%%zmm1 \n" + "vpsubw %%zmm1,%%zmm5,%%zmm1 \n" + "vpermw %%zmm0,%%zmm1,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%[dst_uv])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [dst_uv] "+r"(dst_uv), + [width] "+r"(temp_width), + [temp] "=&r"(temp) + : [src_uv] "r"(src_uv), + [rev] "m"(kMirrorUVPermute_AVX512) + : "memory", "cc", "xmm0", "xmm1", "xmm5", "k1"); } #endif // HAS_MIRRORUVROW_AVX512BW @@ -6560,140 +7200,216 @@ void SplitUVRow_SSE2(const uint8_t* src_uv, } #endif // HAS_SPLITUVROW_SSE2 -#ifdef HAS_DETILEROW_SSE2 -void DetileRow_SSE2(const uint8_t* src, +#ifdef HAS_DETILEROW_AVX2 +void DetileRow_AVX2(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, int width) { asm volatile( - "1: \n" - "movdqu (%0),%%xmm0 \n" - "sub $0x10,%2 \n" - "lea (%0,%3),%0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "r"(src_tile_stride) // %3 - : "cc", "memory", "xmm0"); -} -#endif // HAS_DETILEROW_SSE2 + "sub $0x20,%2 \n" + "jl 2f \n" -#ifdef HAS_DETILEROW_16_SSE2 -void DetileRow_16_SSE2(const uint16_t* src, - ptrdiff_t src_tile_stride, - uint16_t* dst, - int width) { - asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "lea (%0,%3,2),%0 \n" - "movdqu %%xmm0,(%1) \n" - "movdqu %%xmm1,0x10(%1) \n" - "lea 0x20(%1),%1 \n" - "sub $0x10,%2 \n" - "jg 1b \n" - : "+r"(src), // %0 - "+r"(dst), // %1 - "+r"(width) // %2 - : "r"(src_tile_stride) // %3 - : "cc", "memory", "xmm0", "xmm1"); -} -#endif // HAS_DETILEROW_16_SSE2 - -#ifdef HAS_DETILEROW_16_AVX -void DetileRow_16_AVX(const uint16_t* src, - ptrdiff_t src_tile_stride, - uint16_t* dst, - int width) { - asm volatile( - "1: \n" - "vmovdqu (%0),%%ymm0 \n" + "vmovdqu (%0),%%xmm0 \n" + "vinserti128 $0x1,(%0,%3,1),%%ymm0,%%ymm0 \n" "lea (%0,%3,2),%0 \n" "vmovdqu %%ymm0,(%1) \n" "lea 0x20(%1),%1 \n" - "sub $0x10,%2 \n" - "jg 1b \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%xmm0 \n" + "vmovdqu %%xmm0,(%1) \n" + + "99: \n" "vzeroupper \n" : "+r"(src), // %0 "+r"(dst), // %1 "+r"(width) // %2 : "r"(src_tile_stride) // %3 - : "cc", "memory", "xmm0"); + : "memory", "cc", "xmm0"); } -#endif // HAS_DETILEROW_16_AVX +#endif // HAS_DETILEROW_AVX2 -#ifdef HAS_DETILETOYUY2_SSE2 -// Read 16 Y, 8 UV, and write 8 YUYV. -void DetileToYUY2_SSE2(const uint8_t* src_y, +#ifdef HAS_DETILEROW_16_AVX2 +void DetileRow_16_AVX2(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width) { + asm volatile( + "sub $0x20,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu (%0,%3,2),%%ymm1 \n" + "lea (%0,%3,4),%0 \n" + "vmovdqu %%ymm0,(%1) \n" + "vmovdqu %%ymm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : "r"(src_tile_stride) // %3 + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_DETILEROW_16_AVX2 + +#ifdef HAS_DETILETOYUY2_AVX2 +// Read 32 Y, 16 UV, and write 16 YUYV (64 bytes). +void DetileToYUY2_AVX2(const uint8_t* src_y, ptrdiff_t src_y_tile_stride, const uint8_t* src_uv, ptrdiff_t src_uv_tile_stride, uint8_t* dst_yuy2, int width) { asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" // Load 16 Y - "sub $0x10,%3 \n" - "lea (%0,%4),%0 \n" - "movdqu (%1),%%xmm1 \n" // Load 8 UV - "lea (%1,%5),%1 \n" - "movdqu %%xmm0,%%xmm2 \n" - "punpcklbw %%xmm1,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm2 \n" - "movdqu %%xmm0,(%2) \n" - "movdqu %%xmm2,0x10(%2) \n" - "lea 0x20(%2),%2 \n" + "vmovdqu (%0),%%xmm0 \n" + "vinserti128 $0x1,(%0,%4,1),%%ymm0,%%ymm0 \n" + "lea (%0,%4,2),%0 \n" + "vmovdqu (%1),%%xmm1 \n" + "vinserti128 $0x1,(%1,%5,1),%%ymm1,%%ymm1 \n" + "lea (%1,%5,2),%1 \n" + "vpunpcklbw %%ymm1,%%ymm0,%%ymm2 \n" + "vpunpckhbw %%ymm1,%%ymm0,%%ymm3 \n" + "vperm2i128 $0x20,%%ymm3,%%ymm2,%%ymm0 \n" + "vperm2i128 $0x31,%%ymm3,%%ymm2,%%ymm1 \n" + "vmovdqu %%ymm0,(%2) \n" + "vmovdqu %%ymm1,0x20(%2) \n" + "lea 0x40(%2),%2 \n" + "subl $0x20,%3 \n" "jg 1b \n" - : "+r"(src_y), // %0 - "+r"(src_uv), // %1 - "+r"(dst_yuy2), // %2 - "+r"(width) // %3 - : "r"(src_y_tile_stride), // %4 - "r"(src_uv_tile_stride) // %5 - : "cc", "memory", "xmm0", "xmm1", "xmm2" // Clobber list - ); + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(src_uv), // %1 + "+r"(dst_yuy2), // %2 + "+rm"(width) // %3 + : "r"(src_y_tile_stride), // %4 + "r"(src_uv_tile_stride) // %5 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } -#endif // HAS_DETILETOYUY2_SSE2 +#endif // HAS_DETILETOYUY2_AVX2 -#ifdef HAS_DETILESPLITUVROW_SSSE3 -// TODO(greenjustin): Look into generating these constants instead of loading -// them since this can cause branch mispredicts for fPIC code on 32-bit -// machines. +#ifdef HAS_UNPACKMT2T_AVX2 +void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) { + static const uint32_t kMaskC0 = 0xc0c0c0c0u; + if (size >= 80) { + asm volatile( + "vpbroadcastd %[c0],%%ymm6 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%xmm0 \n" + "vpsllw $2,%%xmm0,%%xmm1 \n" + "vinserti128 $0x1,%%xmm0,%%ymm1,%%ymm1 \n" + "vpsllw $4,%%ymm1,%%ymm0 \n" + "vpand %%ymm6,%%ymm0,%%ymm0 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vmovdqu 0x10(%[src]),%%ymm2 \n" + "vmovdqu 0x30(%[src]),%%ymm3 \n" + "lea 0x50(%[src]),%[src] \n" + "vpsrlw $2,%%ymm2,%%ymm4 \n" + "vpsrlw $2,%%ymm3,%%ymm5 \n" + "vpandn %%ymm4,%%ymm6,%%ymm4 \n" + "vpandn %%ymm5,%%ymm6,%%ymm5 \n" + "vpor %%ymm4,%%ymm0,%%ymm0 \n" + "vpor %%ymm5,%%ymm1,%%ymm1 \n" + + "vpunpcklbw %%ymm2,%%ymm0,%%ymm4 \n" + "vpunpckhbw %%ymm2,%%ymm0,%%ymm5 \n" + "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm0 \n" + "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm2 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "vmovdqu %%ymm2,0x20(%[dst]) \n" + + "vpunpcklbw %%ymm3,%%ymm1,%%ymm4 \n" + "vpunpckhbw %%ymm3,%%ymm1,%%ymm5 \n" + "vperm2i128 $0x20,%%ymm5,%%ymm4,%%ymm1 \n" + "vperm2i128 $0x31,%%ymm5,%%ymm4,%%ymm3 \n" + "vmovdqu %%ymm1,0x40(%[dst]) \n" + "vmovdqu %%ymm3,0x60(%[dst]) \n" + + "lea 0x80(%[dst]),%[dst] \n" + "sub $0x50,%[size] \n" + "cmp $0x50,%[size] \n" + "jae 1b \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [size] "+r"(size) + : [c0] "m"(kMaskC0) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6"); + } + if (size > 0) { + UnpackMT2T_C(src, dst, size); + } +} +#endif // HAS_UNPACKMT2T_AVX2 + +#ifdef HAS_DETILESPLITUVROW_AVX2 static const uvec8 kDeinterlaceUV = {0, 2, 4, 6, 8, 10, 12, 14, 1, 3, 5, 7, 9, 11, 13, 15}; -// TODO(greenjustin): Research alternatives to pshufb, since pshufb can be very -// slow on older SSE2 processors. -void DetileSplitUVRow_SSSE3(const uint8_t* src_uv, - ptrdiff_t src_tile_stride, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { +void DetileSplitUVRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { asm volatile( - "movdqu %4,%%xmm1 \n" + "vbroadcasti128 %4,%%ymm1 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "lea (%0, %5),%0 \n" - "pshufb %%xmm1,%%xmm0 \n" - "movq %%xmm0,(%1) \n" - "lea 0x8(%1),%1 \n" - "movhps %%xmm0,(%2) \n" - "lea 0x8(%2),%2 \n" - "sub $0x10,%3 \n" - "jg 1b \n" + "vmovdqu (%0),%%xmm0 \n" + "vinserti128 $0x1,(%0,%5,1),%%ymm0,%%ymm0 \n" + "lea (%0,%5,2),%0 \n" + "vpshufb %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%1) \n" + "lea 0x10(%1),%1 \n" + "vextracti128 $0x1,%%ymm0,(%2) \n" + "lea 0x10(%2),%2 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "jle 99f \n" + "vmovdqu (%0),%%xmm0 \n" + "vpshufb %%xmm1,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0,(%1) \n" + "vmovhps %%xmm0,(%2) \n" + + "99: \n" + "vzeroupper \n" : "+r"(src_uv), // %0 "+r"(dst_u), // %1 "+r"(dst_v), // %2 "+r"(width) // %3 : "m"(kDeinterlaceUV), // %4 "r"(src_tile_stride) // %5 - : "cc", "memory", "xmm0", "xmm1"); + : "memory", "cc", "xmm0", "xmm1"); } -#endif // HAS_DETILESPLITUVROW_SSSE3 +#endif // HAS_DETILESPLITUVROW_AVX2 #ifdef HAS_MERGEUVROW_AVX512BW void MergeUVRow_AVX512BW(const uint8_t* src_u, @@ -6816,6 +7532,80 @@ void MergeUVRow_16_AVX2(const uint16_t* src_u, } #endif // HAS_MERGEUVROW_16_AVX2 +#ifdef HAS_MERGEUVROW_16_AVX512BW +void MergeUVRow_16_AVX512BW(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + uintptr_t temp; + asm volatile( + "vmovd %[shift_u],%%xmm3 \n" + "vmovd %[shift_v],%%xmm4 \n" + "sub %0,%1 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + // 32 pixels per loop. + LABELALIGN + "1: \n" + "vpmovzxwd (%0),%%zmm0 \n" + "vpmovzxwd 0x20(%0),%%zmm2 \n" + "vpmovzxwd 0x00(%0,%1,1),%%zmm1 \n" + "vpmovzxwd 0x20(%0,%1,1),%%zmm5 \n" + "lea 0x40(%0),%0 \n" + "vpsllw %%xmm3,%%zmm0,%%zmm0 \n" + "vpsllw %%xmm3,%%zmm2,%%zmm2 \n" + "vpslld %%xmm4,%%zmm1,%%zmm1 \n" + "vpslld %%xmm4,%%zmm5,%%zmm5 \n" + "vpord %%zmm0,%%zmm1,%%zmm0 \n" + "vpord %%zmm2,%%zmm5,%%zmm2 \n" + "vmovdqu32 %%zmm0,(%2) \n" + "vmovdqu32 %%zmm2,0x40(%2) \n" + "lea 0x80(%2),%2 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "je 99f \n" + "cmp $0x10,%3 \n" + "jl 3f \n" + "vpmovzxwd (%0),%%zmm0 \n" + "vpmovzxwd 0x00(%0,%1,1),%%zmm1 \n" + "lea 0x20(%0),%0 \n" + "vpsllw %%xmm3,%%zmm0,%%zmm0 \n" + "vpslld %%xmm4,%%zmm1,%%zmm1 \n" + "vpord %%zmm0,%%zmm1,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%2) \n" + "lea 0x40(%2),%2 \n" + "sub $0x10,%3 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vpmovzxwd (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmovzxwd 0x00(%0,%1,1),%%zmm1%{%%k1%}%{z%} \n" + "vpsllw %%xmm3,%%zmm0,%%zmm0 \n" + "vpslld %%xmm4,%%zmm1,%%zmm1 \n" + "vpord %%zmm0,%%zmm1,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%2)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_u), // %0 + "+r"(src_v), // %1 + "+r"(dst_uv), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [shift_u] "r"(16 - depth), // %[shift_u] + [shift_v] "r"(32 - depth) // %[shift_v] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "k1"); +} +#endif // HAS_MERGEUVROW_16_AVX512BW + #ifdef HAS_SPLITUVROW_16_AVX2 const uvec8 kSplitUVShuffle16 = {0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; @@ -6959,6 +7749,60 @@ void DivideRow_16_AVX2(const uint16_t* src_y, } #endif // HAS_DIVIDEROW_16_AVX2 +#ifdef HAS_DIVIDEROW_16_AVX512BW +void DivideRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastw %[scale],%%zmm2 \n" + "sub $0x40,%2 \n" + "jl 2f \n" + + // 64 pixels per loop. + LABELALIGN + "1: \n" + "vpmulhuw (%0),%%zmm2,%%zmm0 \n" + "vpmulhuw 0x40(%0),%%zmm2,%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "vmovdqu16 %%zmm1,0x40(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "je 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vpmulhuw (%0),%%zmm2,%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vmovdqu16 %%zmm0,(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x20,%2 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vmovdqu16 %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_y), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : [scale] "r"(scale) // %[scale] + : "memory", "cc", "xmm0", "xmm1", "xmm2", "k1"); +} +#endif // HAS_DIVIDEROW_16_AVX512BW + // Use scale to convert lsb formats to msb, depending how many bits there are: // 32768 = 9 bits // 16384 = 10 bits @@ -7237,6 +8081,72 @@ void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, } #endif // HAS_HALFWIDTHROW_16TO8_AVX2 +#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) || \ + defined(HAS_HALFWIDTHROW_16TO8_AVX2) +void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + int n = (width - 1) & ~15; + if (n > 0) { + HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n); + src_uv += n * 2; + dst_uv += n; + width -= n; + } + HalfWidthRow_16To8_Odd_C(src_uv, src_uv_stride, dst_uv, scale, width); +} +#endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2 + +#ifdef HAS_HALFROW_AVX2 +void HalfRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { + asm volatile( + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpavgb (%0,%3,1),%%ymm0,%%ymm0 \n" + "lea 0x20(%0),%0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x20,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride) // %3 + : "memory", "cc", "xmm0"); +} +#endif // HAS_HALFROW_AVX2 + +#ifdef HAS_HALFROW_16_AVX2 +void HalfRow_16_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { + asm volatile( + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpavgw (%0,%3,2),%%ymm0,%%ymm0 \n" + "lea 0x20(%0),%0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(width) // %2 + : "r"(src_uv_stride) // %3 + : "memory", "cc", "xmm0"); +} +#endif // HAS_HALFROW_16_AVX2 + #ifdef HAS_HALFROW_16TO8_AVX512BW void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, ptrdiff_t src_uv_stride, @@ -7616,7 +8526,7 @@ void SplitRGBRow_AVX2(const uint8_t* src_rgb, } #endif // HAS_SPLITRGBROW_AVX2 -#ifdef HAS_MERGERGBROW_SSSE3 +#ifdef HAS_MERGERGBROW_AVX2 // Shuffle table for converting Planar to RGB. static const uvec8 kMergeRGBShuffle[9] = { {0u, 128u, 128u, 1u, 128u, 128u, 2u, 128u, 128u, 3u, 128u, 128u, 4u, 128u, @@ -7638,58 +8548,69 @@ static const uvec8 kMergeRGBShuffle[9] = { {10u, 128u, 128u, 11u, 128u, 128u, 12u, 128u, 128u, 13u, 128u, 128u, 14u, 128u, 128u, 15u}}; -void MergeRGBRow_SSSE3(const uint8_t* src_r, - const uint8_t* src_g, - const uint8_t* src_b, - uint8_t* dst_rgb, - int width) { +void MergeRGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { asm volatile( + "sub %0,%1 \n" + "sub %0,%2 \n" + + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%1),%%xmm1 \n" - "movdqu (%2),%%xmm2 \n" - "pshufb (%5), %%xmm0 \n" - "pshufb 16(%5), %%xmm1 \n" - "pshufb 32(%5), %%xmm2 \n" - "por %%xmm1,%%xmm0 \n" - "por %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%3) \n" + "vmovdqu (%0),%%ymm3 \n" + "vmovdqu (%0,%1,1),%%ymm4 \n" + "vmovdqu (%0,%2,1),%%ymm5 \n" + "lea 0x20(%0),%0 \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%1),%%xmm1 \n" - "movdqu (%2),%%xmm2 \n" - "pshufb 48(%5), %%xmm0 \n" - "pshufb 64(%5), %%xmm1 \n" - "pshufb 80(%5), %%xmm2 \n" - "por %%xmm1,%%xmm0 \n" - "por %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,16(%3) \n" + "vbroadcasti128 (%5),%%ymm6 \n" + "vbroadcasti128 16(%5),%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm0 \n" + "vpshufb %%ymm7,%%ymm4,%%ymm6 \n" + "vbroadcasti128 32(%5),%%ymm7 \n" + "vpor %%ymm6,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm7,%%ymm5,%%ymm7 \n" + "vpor %%ymm7,%%ymm0,%%ymm0 \n" - "movdqu (%0),%%xmm0 \n" - "movdqu (%1),%%xmm1 \n" - "movdqu (%2),%%xmm2 \n" - "pshufb 96(%5), %%xmm0 \n" - "pshufb 112(%5), %%xmm1 \n" - "pshufb 128(%5), %%xmm2 \n" - "por %%xmm1,%%xmm0 \n" - "por %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,32(%3) \n" + "vbroadcasti128 48(%5),%%ymm6 \n" + "vbroadcasti128 64(%5),%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm1 \n" + "vpshufb %%ymm7,%%ymm4,%%ymm6 \n" + "vbroadcasti128 80(%5),%%ymm7 \n" + "vpor %%ymm6,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm7,%%ymm5,%%ymm7 \n" + "vpor %%ymm7,%%ymm1,%%ymm1 \n" - "lea 0x10(%0),%0 \n" - "lea 0x10(%1),%1 \n" - "lea 0x10(%2),%2 \n" - "lea 0x30(%3),%3 \n" - "sub $0x10,%4 \n" + "vbroadcasti128 96(%5),%%ymm6 \n" + "vbroadcasti128 112(%5),%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm2 \n" + "vpshufb %%ymm7,%%ymm4,%%ymm6 \n" + "vbroadcasti128 128(%5),%%ymm7 \n" + "vpor %%ymm6,%%ymm2,%%ymm2 \n" + "vpshufb %%ymm7,%%ymm5,%%ymm7 \n" + "vpor %%ymm7,%%ymm2,%%ymm2 \n" + + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm6 \n" + "vperm2i128 $0x30,%%ymm0,%%ymm2,%%ymm7 \n" + "vperm2i128 $0x31,%%ymm2,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm6,(%3) \n" + "vmovdqu %%ymm7,0x20(%3) \n" + "vmovdqu %%ymm1,0x40(%3) \n" + "lea 0x60(%3),%3 \n" + "subl $0x20,%4 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_r), // %0 "+r"(src_g), // %1 "+r"(src_b), // %2 "+r"(dst_rgb), // %3 - "+r"(width) // %4 + "+rm"(width) // %4 : "r"(&kMergeRGBShuffle[0]) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); } -#endif // HAS_MERGERGBROW_SSSE3 +#endif // HAS_MERGERGBROW_AVX2 #ifdef HAS_MERGEARGBROW_SSE2 void MergeARGBRow_SSE2(const uint8_t* src_r, @@ -8877,17 +9798,7 @@ void ARGBCopyYToAlphaRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { } #endif // HAS_ARGBCOPYYTOALPHAROW_AVX512BW -#ifdef HAS_SETROW_X86 -void SetRow_X86(uint8_t* dst, uint8_t v8, int width) { - size_t width_tmp = (size_t)(width >> 2); - const uint32_t v32 = v8 * 0x01010101u; // Duplicate byte to all bytes. - asm volatile("rep stosl \n" - : "+D"(dst), // %0 - "+c"(width_tmp) // %1 - : "a"(v32) // %2 - : "memory", "cc"); -} - +#ifdef HAS_SETROW_ERMS void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width) { size_t width_tmp = (size_t)(width); asm volatile("rep stosb \n" @@ -8896,16 +9807,117 @@ void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width) { : "a"(v8) // %2 : "memory", "cc"); } +#endif // HAS_SETROW_ERMS -void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width) { - size_t width_tmp = (size_t)(width); - asm volatile("rep stosl \n" - : "+D"(dst_argb), // %0 - "+c"(width_tmp) // %1 - : "a"(v32) // %2 - : "memory", "cc"); +#ifdef HAS_SETROW_AVX2 +void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width) { + asm volatile( + "vmovd %2,%%xmm0 \n" + "vpbroadcastb %%xmm0,%%ymm0 \n" + "sub $0x40,%1 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu %%ymm0,(%0) \n" + "vmovdqu %%ymm0,0x20(%0) \n" + "lea 0x40(%0),%0 \n" + "sub $0x40,%1 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%1 \n" + "jle 99f \n" + "vmovdqu %%ymm0,(%0) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(dst), // %0 + "+r"(width) // %1 + : "r"((int)v8) // %2 + : "memory", "cc", "xmm0"); } -#endif // HAS_SETROW_X86 +#endif // HAS_SETROW_AVX2 + +#ifdef HAS_ARGBSETROW_AVX2 +void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width) { + asm volatile( + "vmovd %2,%%xmm0 \n" + "vpbroadcastd %%xmm0,%%ymm0 \n" + "sub $0x20,%1 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu %%ymm0,(%0) \n" + "vmovdqu %%ymm0,0x20(%0) \n" + "vmovdqu %%ymm0,0x40(%0) \n" + "vmovdqu %%ymm0,0x60(%0) \n" + "lea 0x80(%0),%0 \n" + "sub $0x20,%1 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%1 \n" + "jle 99f \n" + "cmp $0x10,%1 \n" + "jl 3f \n" + "vmovdqu %%ymm0,(%0) \n" + "vmovdqu %%ymm0,0x20(%0) \n" + "lea 0x40(%0),%0 \n" + "sub $0x10,%1 \n" + "jle 99f \n" + + "3: \n" + "vmovdqu %%ymm0,(%0) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(dst_argb), // %0 + "+r"(width) // %1 + : "r"(v32) // %2 + : "memory", "cc", "xmm0"); +} +#endif // HAS_ARGBSETROW_AVX2 + +#ifdef HAS_COPYROW_16_AVX2 +void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int width) { + asm volatile( + "sub $0x40,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu 0x40(%0),%%ymm2 \n" + "vmovdqu 0x60(%0),%%ymm3 \n" + "lea 0x80(%0),%0 \n" + "vmovdqu %%ymm0,(%1) \n" + "vmovdqu %%ymm1,0x20(%1) \n" + "vmovdqu %%ymm2,0x40(%1) \n" + "vmovdqu %%ymm3,0x60(%1) \n" + "lea 0x80(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu %%ymm0,(%1) \n" + "vmovdqu %%ymm1,0x20(%1) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); +} +#endif // HAS_COPYROW_16_AVX2 #ifdef HAS_YUY2TOYROW_SSE2 void YUY2ToYRow_SSE2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { @@ -9364,92 +10376,323 @@ void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy, } #endif // HAS_YUY2TOYROW_AVX2 -#ifdef HAS_ARGBBLENDROW_SSSE3 -// Shuffle table for isolating alpha. -static const uvec8 kShuffleAlpha = {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, - 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80}; - -// Blend 8 pixels at a time -void ARGBBlendRow_SSSE3(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { +#ifdef HAS_YUY2TOYROW_AVX512BW +void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + uintptr_t temp; asm volatile( - "pcmpeqb %%xmm7,%%xmm7 \n" - "psrlw $0xf,%%xmm7 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "psrlw $0x8,%%xmm6 \n" - "pcmpeqb %%xmm5,%%xmm5 \n" - "psllw $0x8,%%xmm5 \n" - "pcmpeqb %%xmm4,%%xmm4 \n" - "pslld $0x18,%%xmm4 \n" - "sub $0x4,%3 \n" - "jl 49f \n" + "sub $0x40,%2 \n" + "jl 2f \n" - // 4 pixel loop. LABELALIGN - "40: \n" - "movdqu (%0),%%xmm3 \n" - "lea 0x10(%0),%0 \n" - "movdqa %%xmm3,%%xmm0 \n" - "pxor %%xmm4,%%xmm3 \n" - "movdqu (%1),%%xmm2 \n" - "pshufb %4,%%xmm3 \n" - "pand %%xmm6,%%xmm2 \n" - "paddw %%xmm7,%%xmm3 \n" - "pmullw %%xmm3,%%xmm2 \n" - "movdqu (%1),%%xmm1 \n" + "1: \n" + "vmovdqu16 (%0),%%zmm0 \n" + "vmovdqu16 0x40(%0),%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vpmovwb %%zmm0,(%1) \n" + "vpmovwb %%zmm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "je 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vmovdqu16 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpmovwb %%zmm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x20,%2 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpmovwb %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_YUY2TOYROW_AVX512BW + +#ifdef HAS_YUY2TOUVROW_AVX512BW +void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + uintptr_t temp; + asm volatile( + "sub %1,%2 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" + "vpavgb 0x00(%0,%[stride],1),%%zmm0,%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x8,%%zmm0,%%zmm1 \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm1,(%1) \n" + "vpmovdb %%zmm0,0x00(%1,%2,1) \n" "lea 0x10(%1),%1 \n" - "psrlw $0x8,%%xmm1 \n" - "por %%xmm4,%%xmm0 \n" - "pmullw %%xmm3,%%xmm1 \n" - "psrlw $0x8,%%xmm2 \n" - "paddusb %%xmm2,%%xmm0 \n" - "pand %%xmm5,%%xmm1 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "je 99f \n" + "add $0x1,%k3 \n" + "shr $0x1,%k3 \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x00(%0,%[stride],1),%%zmm1%{%%k1%}%{z%} \n" + "vpavgb %%zmm1,%%zmm0,%%zmm0 \n" + "vpsrld $0x8,%%zmm0,%%zmm1 \n" + "vpsrld $0x18,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm1,(%1)%{%%k1%} \n" + "vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_yuy2), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [stride] "r"((ptrdiff_t)(stride_yuy2)) // %[stride] + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_YUY2TOUVROW_AVX512BW + +#ifdef HAS_UYVYTOYROW_AVX512BW +void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + uintptr_t temp; + asm volatile( + "sub $0x40,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu16 (%0),%%zmm0 \n" + "vmovdqu16 0x40(%0),%%zmm1 \n" + "lea 0x80(%0),%0 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpsrlw $0x8,%%zmm1,%%zmm1 \n" + "vpmovwb %%zmm0,(%1) \n" + "vpmovwb %%zmm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x40,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%2 \n" + "je 99f \n" + "cmp $0x20,%2 \n" + "jl 3f \n" + "vmovdqu16 (%0),%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpmovwb %%zmm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x20,%2 \n" + "je 99f \n" + + "3: \n" + "mov $-1,%k[temp] \n" + "bzhi %k2,%k[temp],%k[temp] \n" + "kmovd %k[temp],%%k1 \n" + "vmovdqu16 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vpsrlw $0x8,%%zmm0,%%zmm0 \n" + "vpmovwb %%zmm0,(%1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_y), // %1 + "+r"(width), // %2 + [temp] "=&r"(temp) // %[temp] + : + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_UYVYTOYROW_AVX512BW + +#ifdef HAS_UYVYTOUVROW_AVX512BW +void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + uintptr_t temp; + asm volatile( + "sub %1,%2 \n" + "sub $0x20,%3 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%0),%%zmm0 \n" + "vpavgb 0x00(%0,%[stride],1),%%zmm0,%%zmm0 \n" + "lea 0x40(%0),%0 \n" + "vpmovdb %%zmm0,(%1) \n" + "vpsrld $0x10,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,0x00(%1,%2,1) \n" + "lea 0x10(%1),%1 \n" + "sub $0x20,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%3 \n" + "je 99f \n" + "add $0x1,%k3 \n" + "shr $0x1,%k3 \n" + "mov $-1,%k[temp] \n" + "bzhi %k3,%k[temp],%k[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%0),%%zmm0%{%%k1%}%{z%} \n" + "vmovdqu32 0x00(%0,%[stride],1),%%zmm1%{%%k1%}%{z%} \n" + "vpavgb %%zmm1,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,(%1)%{%%k1%} \n" + "vpsrld $0x10,%%zmm0,%%zmm0 \n" + "vpmovdb %%zmm0,0x00(%1,%2,1)%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uyvy), // %0 + "+r"(dst_u), // %1 + "+r"(dst_v), // %2 + "+r"(width), // %3 + [temp] "=&r"(temp) // %[temp] + : [stride] "r"((ptrdiff_t)(stride_uyvy)) // %[stride] + : "memory", "cc", "xmm0", "xmm1", "k1"); +} +#endif // HAS_UYVYTOUVROW_AVX512BW + +#if defined(HAS_ARGBBLENDROW_AVX2) || defined(HAS_ARGBBLENDROW_AVX512BW) +// Shuffle table for isolating alpha. +SIMD_ALIGNED(static const uvec8 kShuffleAlpha[2]) = { + {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, 11u, 0x80, 11u, 0x80, 15u, 0x80, + 15u, 0x80}, + {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, 11u, 0x80, 11u, 0x80, 15u, 0x80, + 15u, 0x80}}; +#endif // defined(HAS_ARGBBLENDROW_AVX2) || defined(HAS_ARGBBLENDROW_AVX512BW) + +#ifdef HAS_ARGBBLENDROW_AVX2 +// Blend 8 pixels at a time, with 4-pixel and 1-pixel tails. +void ARGBBlendRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n" + "vpsrlw $0xf,%%ymm7,%%ymm7 \n" + "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrlw $0x8,%%ymm6,%%ymm6 \n" + "vpcmpeqb %%ymm5,%%ymm5,%%ymm5 \n" + "vpsllw $0x8,%%ymm5,%%ymm5 \n" + "vpcmpeqb %%ymm4,%%ymm4,%%ymm4 \n" + "vpslld $0x18,%%ymm4,%%ymm4 \n" + "sub $0x8,%3 \n" + "jl 2f \n" + + // 8 pixel loop. + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm3 \n" + "vmovdqu (%1),%%ymm2 \n" + "lea 0x20(%0),%0 \n" + "lea 0x20(%1),%1 \n" + "vpxor %%ymm4,%%ymm3,%%ymm0 \n" + "vpshufb %4,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm7,%%ymm0,%%ymm0 \n" + "vpsrlw $0x8,%%ymm2,%%ymm1 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + "vpmullw %%ymm0,%%ymm2,%%ymm2 \n" + "vpmullw %%ymm0,%%ymm1,%%ymm1 \n" + "vpor %%ymm4,%%ymm3,%%ymm3 \n" + "vpsrlw $0x8,%%ymm2,%%ymm2 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpaddusb %%ymm2,%%ymm3,%%ymm3 \n" + "vpaddusb %%ymm1,%%ymm3,%%ymm3 \n" + "vmovdqu %%ymm3,(%2) \n" + "lea 0x20(%2),%2 \n" + "sub $0x8,%3 \n" + "jge 1b \n" + + "2: \n" + "add $0x4,%3 \n" + "jl 3f \n" + "vmovdqu (%0),%%xmm3 \n" + "vmovdqu (%1),%%xmm2 \n" + "lea 0x10(%0),%0 \n" + "lea 0x10(%1),%1 \n" + "vpxor %%xmm4,%%xmm3,%%xmm0 \n" + "vpshufb %4,%%xmm0,%%xmm0 \n" + "vpaddw %%xmm7,%%xmm0,%%xmm0 \n" + "vpsrlw $0x8,%%xmm2,%%xmm1 \n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + "vpmullw %%xmm0,%%xmm2,%%xmm2 \n" + "vpmullw %%xmm0,%%xmm1,%%xmm1 \n" + "vpor %%xmm4,%%xmm3,%%xmm3 \n" + "vpsrlw $0x8,%%xmm2,%%xmm2 \n" + "vpand %%xmm5,%%xmm1,%%xmm1 \n" + "vpaddusb %%xmm2,%%xmm3,%%xmm3 \n" + "vpaddusb %%xmm1,%%xmm3,%%xmm3 \n" + "vmovdqu %%xmm3,(%2) \n" "lea 0x10(%2),%2 \n" "sub $0x4,%3 \n" - "jge 40b \n" - "49: \n" + "3: \n" "add $0x3,%3 \n" "jl 99f \n" // 1 pixel loop. - "91: \n" - "movd (%0),%%xmm3 \n" + LABELALIGN + "4: \n" + "vmovd (%0),%%xmm3 \n" + "vmovd (%1),%%xmm2 \n" "lea 0x4(%0),%0 \n" - "movdqa %%xmm3,%%xmm0 \n" - "pxor %%xmm4,%%xmm3 \n" - "movd (%1),%%xmm2 \n" - "pshufb %4,%%xmm3 \n" - "pand %%xmm6,%%xmm2 \n" - "paddw %%xmm7,%%xmm3 \n" - "pmullw %%xmm3,%%xmm2 \n" - "movd (%1),%%xmm1 \n" "lea 0x4(%1),%1 \n" - "psrlw $0x8,%%xmm1 \n" - "por %%xmm4,%%xmm0 \n" - "pmullw %%xmm3,%%xmm1 \n" - "psrlw $0x8,%%xmm2 \n" - "paddusb %%xmm2,%%xmm0 \n" - "pand %%xmm5,%%xmm1 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movd %%xmm0,(%2) \n" + "vpxor %%xmm4,%%xmm3,%%xmm0 \n" + "vpshufb %4,%%xmm0,%%xmm0 \n" + "vpaddw %%xmm7,%%xmm0,%%xmm0 \n" + "vpsrlw $0x8,%%xmm2,%%xmm1 \n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + "vpmullw %%xmm0,%%xmm2,%%xmm2 \n" + "vpmullw %%xmm0,%%xmm1,%%xmm1 \n" + "vpor %%xmm4,%%xmm3,%%xmm3 \n" + "vpsrlw $0x8,%%xmm2,%%xmm2 \n" + "vpand %%xmm5,%%xmm1,%%xmm1 \n" + "vpaddusb %%xmm2,%%xmm3,%%xmm3 \n" + "vpaddusb %%xmm1,%%xmm3,%%xmm3 \n" + "vmovd %%xmm3,(%2) \n" "lea 0x4(%2),%2 \n" "sub $0x1,%3 \n" - "jge 91b \n" + "jge 4b \n" + "99: \n" - : "+r"(src_argb), // %0 - "+r"(src_argb1), // %1 - "+r"(dst_argb), // %2 - "+r"(width) // %3 - : "m"(kShuffleAlpha) // %4 + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(src_argb1), // %1 + "+r"(dst_argb), // %2 + "+r"(width) // %3 + : "m"(kShuffleAlpha[0]) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } -#endif // HAS_ARGBBLENDROW_SSSE3 +#endif // HAS_ARGBBLENDROW_AVX2 #ifdef HAS_BLENDPLANEROW_SSSE3 // Blend 8 pixels at a time. @@ -9874,56 +11117,79 @@ void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBUNATTENUATEROW_AVX2 -#ifdef HAS_ARGBGRAYROW_SSSE3 -// Convert 8 ARGB pixels (64 bytes) to 8 Gray ARGB pixels -void ARGBGrayRow_SSSE3(const uint8_t* src_argb, uint8_t* dst_argb, int width) { +#if defined(HAS_ARGBGRAYROW_AVX2) || defined(HAS_ARGBGRAYROW_AVX512BW) || \ + defined(HAS_ARGBGRAYROW_AVX512VNNI) +static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; +#endif // defined(HAS_ARGBGRAYROW_AVX2) || defined(HAS_ARGBGRAYROW_AVX512BW) || + // defined(HAS_ARGBGRAYROW_AVX512VNNI) + +#ifdef HAS_ARGBGRAYROW_AVX2 +void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + static const uint32_t kCoeff = 0x004d961du; // {29, 150, 77, 0} + static const uint32_t kOne16 = 0x00010001u; + static const uint32_t kBias = 32896u; asm volatile( - "movdqa %3,%%xmm4 \n" - "movdqa %4,%%xmm5 \n" + "vpbroadcastd %[coeff],%%ymm2 \n" + "vpcmpeqb %%ymm7,%%ymm7,%%ymm7 \n" + "vpsllw $7,%%ymm7,%%ymm3 \n" + "vpbroadcastb %%xmm3,%%ymm3 \n" // 0x80808080 + "vpslld $24,%%ymm7,%%ymm7 \n" // 0xff000000 + "vpbroadcastd %[one16],%%ymm4 \n" + "vpbroadcastd %[bias],%%ymm5 \n" + "vbroadcasti128 %[shuf],%%ymm6 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" - // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "psubb %%xmm5,%%xmm0 \n" - "psubb %%xmm5,%%xmm1 \n" - "movdqu %%xmm4,%%xmm6 \n" - "pmaddubsw %%xmm0,%%xmm6 \n" - "movdqu %%xmm4,%%xmm0 \n" - "pmaddubsw %%xmm1,%%xmm0 \n" - "phaddw %%xmm0,%%xmm6 \n" - "paddw %%xmm5,%%xmm6 \n" - "psrlw $0x8,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movdqu (%0),%%xmm2 \n" - "movdqu 0x10(%0),%%xmm3 \n" - "lea 0x20(%0),%0 \n" - "psrld $0x18,%%xmm2 \n" - "psrld $0x18,%%xmm3 \n" - "packuswb %%xmm3,%%xmm2 \n" - "packuswb %%xmm2,%%xmm2 \n" - "movdqa %%xmm6,%%xmm3 \n" - "punpcklbw %%xmm6,%%xmm6 \n" - "punpcklbw %%xmm2,%%xmm3 \n" - "movdqa %%xmm6,%%xmm1 \n" - "punpcklwd %%xmm3,%%xmm6 \n" - "punpckhwd %%xmm3,%%xmm1 \n" - "movdqu %%xmm6,(%1) \n" - "movdqu %%xmm1,0x10(%1) \n" - "lea 0x20(%1),%1 \n" - "sub $0x8,%2 \n" - "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "m"(kARGBToYJ), // %3 - "m"(kSub128) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); -} -#endif // HAS_ARGBGRAYROW_SSSE3 + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpxor %%ymm3,%%ymm0,%%ymm1 \n" + "vpmaddubsw %%ymm1,%%ymm2,%%ymm1 \n" + "vpmaddwd %%ymm4,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm5,%%ymm1,%%ymm1 \n" + "vpsrld $8,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm6,%%ymm1,%%ymm1 \n" + "vpand %%ymm7,%%ymm0,%%ymm0 \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" -#ifdef HAS_ARGBSEPIAROW_SSSE3 + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovd (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vpxor %%xmm3,%%xmm0,%%xmm1 \n" + "vpmaddubsw %%xmm1,%%xmm2,%%xmm1 \n" + "vpmaddwd %%xmm4,%%xmm1,%%xmm1 \n" + "vpaddd %%xmm5,%%xmm1,%%xmm1 \n" + "vpsrld $8,%%xmm1,%%xmm1 \n" + "vpshufb %%xmm6,%%xmm1,%%xmm1 \n" + "vpand %%xmm7,%%xmm0,%%xmm0 \n" + "vpor %%xmm1,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width) + : [coeff] "m"(kCoeff), [one16] "m"(kOne16), [bias] "m"(kBias), + [shuf] "m"(kShufGrayDword) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_ARGBGRAYROW_AVX2 + +#if defined(HAS_ARGBSEPIAROW_AVX2) || defined(HAS_ARGBSEPIAROW_AVX512VNNI) // b = (r * 35 + g * 68 + b * 17) >> 7 // g = (r * 45 + g * 88 + b * 22) >> 7 // r = (r * 50 + g * 98 + b * 24) >> 7 @@ -9936,215 +11202,458 @@ static const vec8 kARGBToSepiaG = {22, 88, 45, 0, 22, 88, 45, 0, static const vec8 kARGBToSepiaR = {24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0, 24, 98, 50, 0}; +#endif -// Convert 8 ARGB pixels (32 bytes) to 8 Sepia ARGB pixels. -void ARGBSepiaRow_SSSE3(uint8_t* dst_argb, int width) { +#ifdef HAS_ARGBSEPIAROW_AVX2 +// Convert ARGB pixels to Sepia ARGB pixels. +void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) { asm volatile( - "movdqa %2,%%xmm2 \n" - "movdqa %3,%%xmm3 \n" - "movdqa %4,%%xmm4 \n" + "vbroadcasti128 %2,%%ymm2 \n" + "vbroadcasti128 %3,%%ymm3 \n" + "vbroadcasti128 %4,%%ymm4 \n" + "sub $0x8,%1 \n" + "jl 2f \n" // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm6 \n" - "pmaddubsw %%xmm2,%%xmm0 \n" - "pmaddubsw %%xmm2,%%xmm6 \n" - "phaddw %%xmm6,%%xmm0 \n" - "psrlw $0x7,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movdqu (%0),%%xmm5 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "pmaddubsw %%xmm3,%%xmm5 \n" - "pmaddubsw %%xmm3,%%xmm1 \n" - "phaddw %%xmm1,%%xmm5 \n" - "psrlw $0x7,%%xmm5 \n" - "packuswb %%xmm5,%%xmm5 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "movdqu (%0),%%xmm5 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "pmaddubsw %%xmm4,%%xmm5 \n" - "pmaddubsw %%xmm4,%%xmm1 \n" - "phaddw %%xmm1,%%xmm5 \n" - "psrlw $0x7,%%xmm5 \n" - "packuswb %%xmm5,%%xmm5 \n" - "movdqu (%0),%%xmm6 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "psrld $0x18,%%xmm6 \n" - "psrld $0x18,%%xmm1 \n" - "packuswb %%xmm1,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "punpcklbw %%xmm6,%%xmm5 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpcklwd %%xmm5,%%xmm0 \n" - "punpckhwd %%xmm5,%%xmm1 \n" - "movdqu %%xmm0,(%0) \n" - "movdqu %%xmm1,0x10(%0) \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpmaddubsw %%ymm2,%%ymm0,%%ymm1 \n" + "vpmaddubsw %%ymm3,%%ymm0,%%ymm5 \n" + "vpmaddubsw %%ymm4,%%ymm0,%%ymm6 \n" + "vphaddw %%ymm5,%%ymm1,%%ymm1 \n" + "vphaddw %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrlw $0x7,%%ymm1,%%ymm1 \n" + "vpsrlw $0x7,%%ymm6,%%ymm6 \n" + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm6,%%ymm6,%%ymm6 \n" + "vpshufd $0x01,%%ymm1,%%ymm5 \n" + "vpunpcklbw %%ymm5,%%ymm1,%%ymm1 \n" + "vpsrld $0x18,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm0,%%ymm6,%%ymm6 \n" + "vpunpcklwd %%ymm6,%%ymm1,%%ymm0 \n" + "vmovdqu %%ymm0,(%0) \n" "lea 0x20(%0),%0 \n" "sub $0x8,%1 \n" - "jg 1b \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%1 \n" + "jle 99f \n" + + // 1 pixel remainder loop. + LABELALIGN + "3: \n" + "vmovd (%0),%%xmm0 \n" + "vpmaddubsw %%xmm2,%%xmm0,%%xmm1 \n" + "vpmaddubsw %%xmm3,%%xmm0,%%xmm5 \n" + "vpmaddubsw %%xmm4,%%xmm0,%%xmm6 \n" + "vphaddw %%xmm5,%%xmm1,%%xmm1 \n" + "vphaddw %%xmm6,%%xmm6,%%xmm6 \n" + "vpsrlw $0x7,%%xmm1,%%xmm1 \n" + "vpsrlw $0x7,%%xmm6,%%xmm6 \n" + "vpackuswb %%xmm1,%%xmm1,%%xmm1 \n" + "vpackuswb %%xmm6,%%xmm6,%%xmm6 \n" + "vpshufd $0x01,%%xmm1,%%xmm5 \n" + "vpunpcklbw %%xmm5,%%xmm1,%%xmm1 \n" + "vpsrld $0x18,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vpunpcklbw %%xmm0,%%xmm6,%%xmm6 \n" + "vpunpcklwd %%xmm6,%%xmm1,%%xmm0 \n" + "vmovd %%xmm0,(%0) \n" + "lea 0x04(%0),%0 \n" + "sub $0x1,%1 \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" : "+r"(dst_argb), // %0 "+r"(width) // %1 : "m"(kARGBToSepiaB), // %2 "m"(kARGBToSepiaG), // %3 "m"(kARGBToSepiaR) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6"); } -#endif // HAS_ARGBSEPIAROW_SSSE3 +#endif // HAS_ARGBSEPIAROW_AVX2 -#ifdef HAS_ARGBCOLORMATRIXROW_SSSE3 -// Tranform 8 ARGB pixels (32 bytes) with color matrix. -// Same as Sepia except matrix is provided. -void ARGBColorMatrixRow_SSSE3(const uint8_t* src_argb, - uint8_t* dst_argb, - const int8_t* matrix_argb, - int width) { +#ifdef HAS_ARGBSEPIAROW_AVX512VNNI +void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) { + uintptr_t temp; asm volatile( - "movdqu (%3),%%xmm5 \n" - "pshufd $0x00,%%xmm5,%%xmm2 \n" - "pshufd $0x55,%%xmm5,%%xmm3 \n" - "pshufd $0xaa,%%xmm5,%%xmm4 \n" - "pshufd $0xff,%%xmm5,%%xmm5 \n" + "vpbroadcastd %[kSepiaB],%%zmm2 \n" + "vpbroadcastd %[kSepiaG],%%zmm3 \n" + "vpbroadcastd %[kSepiaR],%%zmm4 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrld $24,%%zmm5,%%zmm6 \n" // 0x000000ff + "vpslld $24,%%zmm5,%%zmm5 \n" // 0xff000000 + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[dst]),%%zmm0 \n" + "vpxord %%zmm1,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vpxord %%zmm8,%%zmm8,%%zmm8 \n" + "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n" + "vpsrld $7,%%zmm1,%%zmm1 \n" + "vpsrld $7,%%zmm7,%%zmm7 \n" + "vpsrld $7,%%zmm8,%%zmm8 \n" + "vpminud %%zmm6,%%zmm7,%%zmm7 \n" + "vpminud %%zmm6,%%zmm8,%%zmm8 \n" + "vpslld $8,%%zmm7,%%zmm7 \n" + "vpslld $16,%%zmm8,%%zmm8 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm8,%%zmm0,%%zmm0 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[dst]),%%zmm0%{%%k1%}%{z%} \n" + "vpxord %%zmm1,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vpxord %%zmm8,%%zmm8,%%zmm8 \n" + "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n" + "vpsrld $7,%%zmm1,%%zmm1 \n" + "vpsrld $7,%%zmm7,%%zmm7 \n" + "vpsrld $7,%%zmm8,%%zmm8 \n" + "vpminud %%zmm6,%%zmm7,%%zmm7 \n" + "vpminud %%zmm6,%%zmm8,%%zmm8 \n" + "vpslld $8,%%zmm7,%%zmm7 \n" + "vpslld $16,%%zmm8,%%zmm8 \n" + "vpandd %%zmm5,%%zmm0,%%zmm0 \n" + "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm8,%%zmm0,%%zmm0 \n" + "vpord %%zmm1,%%zmm0,%%zmm0 \n" + "vmovdqu32 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [width] "+r"(width), [temp] "=&r"(temp) + : [kSepiaB] "m"(kARGBToSepiaB), [kSepiaG] "m"(kARGBToSepiaG), + [kSepiaR] "m"(kARGBToSepiaR) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "k1"); +} +#endif // HAS_ARGBSEPIAROW_AVX512VNNI + +#ifdef HAS_ARGBCOLORMATRIXROW_AVX2 +static const vec8 kShufTranspose4x4 = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; +// Transform ARGB pixels with color matrix. +void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + asm volatile( + "vbroadcasti128 (%3),%%ymm5 \n" + "vbroadcasti128 %4,%%ymm7 \n" + "vpshufd $0x00,%%ymm5,%%ymm2 \n" + "vpshufd $0x55,%%ymm5,%%ymm3 \n" + "vpshufd $0xaa,%%ymm5,%%ymm4 \n" + "vpshufd $0xff,%%ymm5,%%ymm5 \n" + "sub $0x8,%2 \n" + "jl 2f \n" // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm7 \n" - "pmaddubsw %%xmm2,%%xmm0 \n" - "pmaddubsw %%xmm2,%%xmm7 \n" - "movdqu (%0),%%xmm6 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "pmaddubsw %%xmm3,%%xmm6 \n" - "pmaddubsw %%xmm3,%%xmm1 \n" - "phaddsw %%xmm7,%%xmm0 \n" - "phaddsw %%xmm1,%%xmm6 \n" - "psraw $0x6,%%xmm0 \n" - "psraw $0x6,%%xmm6 \n" - "packuswb %%xmm0,%%xmm0 \n" - "packuswb %%xmm6,%%xmm6 \n" - "punpcklbw %%xmm6,%%xmm0 \n" - "movdqu (%0),%%xmm1 \n" - "movdqu 0x10(%0),%%xmm7 \n" - "pmaddubsw %%xmm4,%%xmm1 \n" - "pmaddubsw %%xmm4,%%xmm7 \n" - "phaddsw %%xmm7,%%xmm1 \n" - "movdqu (%0),%%xmm6 \n" - "movdqu 0x10(%0),%%xmm7 \n" - "pmaddubsw %%xmm5,%%xmm6 \n" - "pmaddubsw %%xmm5,%%xmm7 \n" - "phaddsw %%xmm7,%%xmm6 \n" - "psraw $0x6,%%xmm1 \n" - "psraw $0x6,%%xmm6 \n" - "packuswb %%xmm1,%%xmm1 \n" - "packuswb %%xmm6,%%xmm6 \n" - "punpcklbw %%xmm6,%%xmm1 \n" - "movdqa %%xmm0,%%xmm6 \n" - "punpcklwd %%xmm1,%%xmm0 \n" - "punpckhwd %%xmm1,%%xmm6 \n" - "movdqu %%xmm0,(%1) \n" - "movdqu %%xmm6,0x10(%1) \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" "lea 0x20(%0),%0 \n" + "vpmaddubsw %%ymm2,%%ymm0,%%ymm1 \n" + "vpmaddubsw %%ymm3,%%ymm0,%%ymm6 \n" + "vphaddsw %%ymm6,%%ymm1,%%ymm1 \n" + "vpmaddubsw %%ymm4,%%ymm0,%%ymm6 \n" + "vpmaddubsw %%ymm5,%%ymm0,%%ymm0 \n" + "vphaddsw %%ymm0,%%ymm6,%%ymm6 \n" + "vpsraw $0x6,%%ymm1,%%ymm1 \n" + "vpsraw $0x6,%%ymm6,%%ymm6 \n" + "vpackuswb %%ymm6,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm7,%%ymm1,%%ymm1 \n" + "vmovdqu %%ymm1,(%1) \n" "lea 0x20(%1),%1 \n" "sub $0x8,%2 \n" - "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(dst_argb), // %1 - "+r"(width) // %2 - : "r"(matrix_argb) // %3 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); -} -#endif // HAS_ARGBCOLORMATRIXROW_SSSE3 + "jge 1b \n" -#ifdef HAS_ARGBQUANTIZEROW_SSE2 -// Quantize 4 ARGB pixels (16 bytes). -void ARGBQuantizeRow_SSE2(uint8_t* dst_argb, + "2: \n" + "add $0x8,%2 \n" + "jle 99f \n" + + // 1 pixel remainder loop. + LABELALIGN + "3: \n" + "vmovd (%0),%%xmm0 \n" + "lea 0x04(%0),%0 \n" + "vpmaddubsw %%xmm2,%%xmm0,%%xmm1 \n" + "vpmaddubsw %%xmm3,%%xmm0,%%xmm6 \n" + "vphaddsw %%xmm6,%%xmm1,%%xmm1 \n" + "vpmaddubsw %%xmm4,%%xmm0,%%xmm6 \n" + "vpmaddubsw %%xmm5,%%xmm0,%%xmm0 \n" + "vphaddsw %%xmm0,%%xmm6,%%xmm6 \n" + "vpsraw $0x6,%%xmm1,%%xmm1 \n" + "vpsraw $0x6,%%xmm6,%%xmm6 \n" + "vpackuswb %%xmm6,%%xmm1,%%xmm1 \n" + "vpshufb %%xmm7,%%xmm1,%%xmm1 \n" + "vmovd %%xmm1,(%1) \n" + "lea 0x04(%1),%1 \n" + "sub $0x1,%2 \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_argb), // %0 + "+r"(dst_argb), // %1 + "+r"(width) // %2 + : "r"(matrix_argb), // %3 + "m"(kShufTranspose4x4) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#endif // HAS_ARGBCOLORMATRIXROW_AVX2 + +#ifdef HAS_ARGBCOLORMATRIXROW_AVX512VNNI +void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastd (%[mat]),%%zmm2 \n" + "vpbroadcastd 0x04(%[mat]),%%zmm3 \n" + "vpbroadcastd 0x08(%[mat]),%%zmm4 \n" + "vpbroadcastd 0x0c(%[mat]),%%zmm5 \n" + "vpternlogd $0xff,%%zmm6,%%zmm6,%%zmm6 \n" + "vpsrld $24,%%zmm6,%%zmm6 \n" // 255 + "vpxord %%zmm9,%%zmm9,%%zmm9 \n" // 0 + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpxord %%zmm1,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vpxord %%zmm8,%%zmm8,%%zmm8 \n" + "vpxord %%zmm10,%%zmm10,%%zmm10 \n" + "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n" + "vpdpbusd %%zmm5,%%zmm0,%%zmm10 \n" + "vpsrad $6,%%zmm1,%%zmm1 \n" + "vpsrad $6,%%zmm7,%%zmm7 \n" + "vpsrad $6,%%zmm8,%%zmm8 \n" + "vpsrad $6,%%zmm10,%%zmm10 \n" + "vpmaxsd %%zmm9,%%zmm1,%%zmm1 \n" + "vpmaxsd %%zmm9,%%zmm7,%%zmm7 \n" + "vpmaxsd %%zmm9,%%zmm8,%%zmm8 \n" + "vpmaxsd %%zmm9,%%zmm10,%%zmm10 \n" + "vpminud %%zmm6,%%zmm1,%%zmm1 \n" + "vpminud %%zmm6,%%zmm7,%%zmm7 \n" + "vpminud %%zmm6,%%zmm8,%%zmm8 \n" + "vpminud %%zmm6,%%zmm10,%%zmm10 \n" + "vpslld $8,%%zmm7,%%zmm7 \n" + "vpslld $16,%%zmm8,%%zmm8 \n" + "vpslld $24,%%zmm10,%%zmm10 \n" + "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm10,%%zmm8,%%zmm8 \n" + "vpord %%zmm8,%%zmm1,%%zmm1 \n" + "vmovdqu32 %%zmm1,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vmovdqu32 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpxord %%zmm1,%%zmm1,%%zmm1 \n" + "vpxord %%zmm7,%%zmm7,%%zmm7 \n" + "vpxord %%zmm8,%%zmm8,%%zmm8 \n" + "vpxord %%zmm10,%%zmm10,%%zmm10 \n" + "vpdpbusd %%zmm2,%%zmm0,%%zmm1 \n" + "vpdpbusd %%zmm3,%%zmm0,%%zmm7 \n" + "vpdpbusd %%zmm4,%%zmm0,%%zmm8 \n" + "vpdpbusd %%zmm5,%%zmm0,%%zmm10 \n" + "vpsrad $6,%%zmm1,%%zmm1 \n" + "vpsrad $6,%%zmm7,%%zmm7 \n" + "vpsrad $6,%%zmm8,%%zmm8 \n" + "vpsrad $6,%%zmm10,%%zmm10 \n" + "vpmaxsd %%zmm9,%%zmm1,%%zmm1 \n" + "vpmaxsd %%zmm9,%%zmm7,%%zmm7 \n" + "vpmaxsd %%zmm9,%%zmm8,%%zmm8 \n" + "vpmaxsd %%zmm9,%%zmm10,%%zmm10 \n" + "vpminud %%zmm6,%%zmm1,%%zmm1 \n" + "vpminud %%zmm6,%%zmm7,%%zmm7 \n" + "vpminud %%zmm6,%%zmm8,%%zmm8 \n" + "vpminud %%zmm6,%%zmm10,%%zmm10 \n" + "vpslld $8,%%zmm7,%%zmm7 \n" + "vpslld $16,%%zmm8,%%zmm8 \n" + "vpslld $24,%%zmm10,%%zmm10 \n" + "vpord %%zmm7,%%zmm1,%%zmm1 \n" + "vpord %%zmm10,%%zmm8,%%zmm8 \n" + "vpord %%zmm8,%%zmm1,%%zmm1 \n" + "vmovdqu32 %%zmm1,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+r"(width), + [temp] "=&r"(temp) + : [mat] "r"(matrix_argb) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7", "xmm8", "xmm9", "xmm10", "k1"); +} +#endif // HAS_ARGBCOLORMATRIXROW_AVX512VNNI + +#ifdef HAS_ARGBQUANTIZEROW_AVX2 +// Quantize ARGB pixels. +void ARGBQuantizeRow_AVX2(uint8_t* dst_argb, int scale, int interval_size, int interval_offset, int width) { asm volatile( - "movd %2,%%xmm2 \n" - "movd %3,%%xmm3 \n" - "movd %4,%%xmm4 \n" - "pshuflw $0x40,%%xmm2,%%xmm2 \n" - "pshufd $0x44,%%xmm2,%%xmm2 \n" - "pshuflw $0x40,%%xmm3,%%xmm3 \n" - "pshufd $0x44,%%xmm3,%%xmm3 \n" - "pshuflw $0x40,%%xmm4,%%xmm4 \n" - "pshufd $0x44,%%xmm4,%%xmm4 \n" - "pxor %%xmm5,%%xmm5 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "pslld $0x18,%%xmm6 \n" + "vpbroadcastw %2,%%ymm2 \n" + "vpbroadcastw %3,%%ymm3 \n" + "vpbroadcastw %4,%%ymm4 \n" + "vpxor %%ymm5,%%ymm5,%%ymm5 \n" + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpslld $0x18,%%ymm6,%%ymm6 \n" + "sub $0x8,%1 \n" + "jl 2f \n" - // 4 pixel loop. + // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "pmulhuw %%xmm2,%%xmm0 \n" - "movdqu (%0),%%xmm1 \n" - "punpckhbw %%xmm5,%%xmm1 \n" - "pmulhuw %%xmm2,%%xmm1 \n" - "pmullw %%xmm3,%%xmm0 \n" - "movdqu (%0),%%xmm7 \n" - "pmullw %%xmm3,%%xmm1 \n" - "pand %%xmm6,%%xmm7 \n" - "paddw %%xmm4,%%xmm0 \n" - "paddw %%xmm4,%%xmm1 \n" - "packuswb %%xmm1,%%xmm0 \n" - "por %%xmm7,%%xmm0 \n" - "movdqu %%xmm0,(%0) \n" - "lea 0x10(%0),%0 \n" - "sub $0x4,%1 \n" - "jg 1b \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpunpckhbw %%ymm5,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm0,%%ymm7 \n" + "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n" + "vpmullw %%ymm3,%%ymm0,%%ymm0 \n" + "vpmullw %%ymm3,%%ymm1,%%ymm1 \n" + "vpaddw %%ymm4,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm4,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vpandn %%ymm0,%%ymm6,%%ymm0 \n" + "vpor %%ymm7,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%0) \n" + "lea 0x20(%0),%0 \n" + "sub $0x8,%1 \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%1 \n" + "jle 99f \n" + + // 1 pixel remainder loop. + LABELALIGN + "3: \n" + "vmovd (%0),%%xmm0 \n" + "vpand %%xmm6,%%xmm0,%%xmm7 \n" + "vpunpcklbw %%xmm5,%%xmm0,%%xmm0 \n" + "vpmulhuw %%xmm2,%%xmm0,%%xmm0 \n" + "vpmullw %%xmm3,%%xmm0,%%xmm0 \n" + "vpaddw %%xmm4,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vpandn %%xmm0,%%xmm6,%%xmm0 \n" + "vpor %%xmm7,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%0) \n" + "lea 0x04(%0),%0 \n" + "sub $0x1,%1 \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" : "+r"(dst_argb), // %0 "+r"(width) // %1 - : "r"(scale), // %2 - "r"(interval_size), // %3 - "r"(interval_offset) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + : "m"(scale), // %2 + "m"(interval_size), // %3 + "m"(interval_offset) // %4 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); } -#endif // HAS_ARGBQUANTIZEROW_SSE2 +#endif // HAS_ARGBQUANTIZEROW_AVX2 -#ifdef HAS_ARGBSHADEROW_SSE2 -// Shade 4 pixels at a time by specified value. -void ARGBShadeRow_SSE2(const uint8_t* src_argb, +#ifdef HAS_ARGBSHADEROW_AVX2 +// Shade ARGB pixels by specified value. +void ARGBShadeRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width, uint32_t value) { asm volatile( - "movd %3,%%xmm2 \n" - "punpcklbw %%xmm2,%%xmm2 \n" - "punpcklqdq %%xmm2,%%xmm2 \n" + "vpbroadcastd %3,%%ymm2 \n" + "vpunpcklbw %%ymm2,%%ymm2,%%ymm2 \n" + "sub $0x8,%2 \n" + "jl 2f \n" - // 4 pixel loop. + // 8 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "lea 0x10(%0),%0 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpcklbw %%xmm0,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm1 \n" - "pmulhuw %%xmm2,%%xmm0 \n" - "pmulhuw %%xmm2,%%xmm1 \n" - "psrlw $0x8,%%xmm0 \n" - "psrlw $0x8,%%xmm1 \n" - "packuswb %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x4,%2 \n" - "jg 1b \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "lea 0x20(%0),%0 \n" + "vpunpckhbw %%ymm0,%%ymm0,%%ymm1 \n" + "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm0,%%ymm0 \n" + "vpmulhuw %%ymm2,%%ymm1,%%ymm1 \n" + "vpsrlw $0x8,%%ymm0,%%ymm0 \n" + "vpsrlw $0x8,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x8,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%2 \n" + "jle 99f \n" + + // 1 pixel remainder loop. + LABELALIGN + "3: \n" + "vmovd (%0),%%xmm0 \n" + "lea 0x04(%0),%0 \n" + "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n" + "vpmulhuw %%xmm2,%%xmm0,%%xmm0 \n" + "vpsrlw $0x8,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%1) \n" + "lea 0x04(%1),%1 \n" + "sub $0x1,%2 \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" : "+r"(src_argb), // %0 "+r"(dst_argb), // %1 "+r"(width) // %2 - : "r"(value) // %3 + : "m"(value) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2"); } -#endif // HAS_ARGBSHADEROW_SSE2 +#endif // HAS_ARGBSHADEROW_AVX2 #ifdef HAS_ARGBMULTIPLYROW_SSE2 // Multiply 2 rows of ARGB pixels together, 4 pixels at a time. @@ -10341,12 +11850,12 @@ void ARGBSubtractRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBSUBTRACTROW_AVX2 -#ifdef HAS_SOBELXROW_SSE2 +#ifdef HAS_SOBELXROW_AVX2 // SobelX as a matrix is // -1 0 1 // -2 0 2 // -1 0 1 -void SobelXRow_SSE2(const uint8_t* src_y0, +void SobelXRow_AVX2(const uint8_t* src_y0, const uint8_t* src_y1, const uint8_t* src_y2, uint8_t* dst_sobelx, @@ -10355,440 +11864,201 @@ void SobelXRow_SSE2(const uint8_t* src_y0, "sub %0,%1 \n" "sub %0,%2 \n" "sub %0,%3 \n" - "pxor %%xmm5,%%xmm5 \n" - // 8 pixel loop. + // 16 pixel loop. LABELALIGN - "1: \n" - "movq (%0),%%xmm0 \n" - "movq 0x2(%0),%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "psubw %%xmm1,%%xmm0 \n" - "movq 0x00(%0,%1,1),%%xmm1 \n" - "movq 0x02(%0,%1,1),%%xmm2 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "psubw %%xmm2,%%xmm1 \n" - "movq 0x00(%0,%2,1),%%xmm2 \n" - "movq 0x02(%0,%2,1),%%xmm3 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "punpcklbw %%xmm5,%%xmm3 \n" - "psubw %%xmm3,%%xmm2 \n" - "paddw %%xmm2,%%xmm0 \n" - "paddw %%xmm1,%%xmm0 \n" - "paddw %%xmm1,%%xmm0 \n" - "pxor %%xmm1,%%xmm1 \n" - "psubw %%xmm0,%%xmm1 \n" - "pmaxsw %%xmm1,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movq %%xmm0,0x00(%0,%3,1) \n" - "lea 0x8(%0),%0 \n" - "sub $0x8,%4 \n" + "1: \n" + "vpmovzxbw (%0),%%ymm0 \n" + "vpmovzxbw 0x02(%0),%%ymm1 \n" + "vpsubw %%ymm1,%%ymm0,%%ymm0 \n" + "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n" + "vpmovzxbw 0x02(%0,%1,1),%%ymm2 \n" + "vpsubw %%ymm2,%%ymm1,%%ymm1 \n" + "vpmovzxbw 0x00(%0,%2,1),%%ymm2 \n" + "vpmovzxbw 0x02(%0,%2,1),%%ymm3 \n" + "vpsubw %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm2,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm1,%%ymm0,%%ymm0 \n" + "vpabsw %%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,0x00(%0,%3,1) \n" + "lea 0x10(%0),%0 \n" + "sub $0x10,%4 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_y0), // %0 "+r"(src_y1), // %1 "+r"(src_y2), // %2 "+r"(dst_sobelx), // %3 "+r"(width) // %4 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } -#endif // HAS_SOBELXROW_SSE2 +#endif // HAS_SOBELXROW_AVX2 -#ifdef HAS_SOBELYROW_SSE2 +#ifdef HAS_SOBELYROW_AVX2 // SobelY as a matrix is // -1 -2 -1 // 0 0 0 // 1 2 1 -void SobelYRow_SSE2(const uint8_t* src_y0, +void SobelYRow_AVX2(const uint8_t* src_y0, const uint8_t* src_y1, uint8_t* dst_sobely, int width) { asm volatile( "sub %0,%1 \n" "sub %0,%2 \n" - "pxor %%xmm5,%%xmm5 \n" - // 8 pixel loop. + // 16 pixel loop. LABELALIGN - "1: \n" - "movq (%0),%%xmm0 \n" - "movq 0x00(%0,%1,1),%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "psubw %%xmm1,%%xmm0 \n" - "movq 0x1(%0),%%xmm1 \n" - "movq 0x01(%0,%1,1),%%xmm2 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "psubw %%xmm2,%%xmm1 \n" - "movq 0x2(%0),%%xmm2 \n" - "movq 0x02(%0,%1,1),%%xmm3 \n" - "punpcklbw %%xmm5,%%xmm2 \n" - "punpcklbw %%xmm5,%%xmm3 \n" - "psubw %%xmm3,%%xmm2 \n" - "paddw %%xmm2,%%xmm0 \n" - "paddw %%xmm1,%%xmm0 \n" - "paddw %%xmm1,%%xmm0 \n" - "pxor %%xmm1,%%xmm1 \n" - "psubw %%xmm0,%%xmm1 \n" - "pmaxsw %%xmm1,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movq %%xmm0,0x00(%0,%2,1) \n" - "lea 0x8(%0),%0 \n" - "sub $0x8,%3 \n" + "1: \n" + "vpmovzxbw (%0),%%ymm0 \n" + "vpmovzxbw 0x00(%0,%1,1),%%ymm1 \n" + "vpsubw %%ymm1,%%ymm0,%%ymm0 \n" + "vpmovzxbw 0x01(%0),%%ymm1 \n" + "vpmovzxbw 0x01(%0,%1,1),%%ymm2 \n" + "vpsubw %%ymm2,%%ymm1,%%ymm1 \n" + "vpmovzxbw 0x02(%0),%%ymm2 \n" + "vpmovzxbw 0x02(%0,%1,1),%%ymm3 \n" + "vpsubw %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm2,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddw %%ymm1,%%ymm0,%%ymm0 \n" + "vpabsw %%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,0x00(%0,%2,1) \n" + "lea 0x10(%0),%0 \n" + "sub $0x10,%3 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_y0), // %0 "+r"(src_y1), // %1 "+r"(dst_sobely), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } -#endif // HAS_SOBELYROW_SSE2 +#endif // HAS_SOBELYROW_AVX2 -#ifdef HAS_SOBELROW_SSE2 +#ifdef HAS_SOBELROW_AVX2 // Adds Sobel X and Sobel Y and stores Sobel into ARGB. // A = 255 // R = Sobel // G = Sobel // B = Sobel -void SobelRow_SSE2(const uint8_t* src_sobelx, +void SobelRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width) { asm volatile( "sub %0,%1 \n" - "pcmpeqb %%xmm5,%%xmm5 \n" - "pslld $0x18,%%xmm5 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpslld $0x18,%%ymm5,%%ymm5 \n" - // 8 pixel loop. + // 16 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x00(%0,%1,1),%%xmm1 \n" + "1: \n" + "vmovdqu (%0),%%xmm0 \n" + "vpaddusb 0x00(%0,%1,1),%%xmm0,%%xmm0 \n" "lea 0x10(%0),%0 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movdqa %%xmm0,%%xmm2 \n" - "punpcklbw %%xmm0,%%xmm2 \n" - "punpckhbw %%xmm0,%%xmm0 \n" - "movdqa %%xmm2,%%xmm1 \n" - "punpcklwd %%xmm2,%%xmm1 \n" - "punpckhwd %%xmm2,%%xmm2 \n" - "por %%xmm5,%%xmm1 \n" - "por %%xmm5,%%xmm2 \n" - "movdqa %%xmm0,%%xmm3 \n" - "punpcklwd %%xmm0,%%xmm3 \n" - "punpckhwd %%xmm0,%%xmm0 \n" - "por %%xmm5,%%xmm3 \n" - "por %%xmm5,%%xmm0 \n" - "movdqu %%xmm1,(%2) \n" - "movdqu %%xmm2,0x10(%2) \n" - "movdqu %%xmm3,0x20(%2) \n" - "movdqu %%xmm0,0x30(%2) \n" + "vpshufd $0xd8,%%xmm0,%%xmm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n" + "vpunpcklwd %%ymm0,%%ymm0,%%ymm1 \n" + "vpunpckhwd %%ymm0,%%ymm0,%%ymm0 \n" + "vpor %%ymm5,%%ymm1,%%ymm1 \n" + "vpor %%ymm5,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm1,(%2) \n" + "vmovdqu %%ymm0,0x20(%2) \n" "lea 0x40(%2),%2 \n" "sub $0x10,%3 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_sobelx), // %0 "+r"(src_sobely), // %1 "+r"(dst_argb), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm5"); } -#endif // HAS_SOBELROW_SSE2 +#endif // HAS_SOBELROW_AVX2 -#ifdef HAS_SOBELTOPLANEROW_SSE2 +#ifdef HAS_SOBELTOPLANEROW_AVX2 // Adds Sobel X and Sobel Y and stores Sobel into a plane. -void SobelToPlaneRow_SSE2(const uint8_t* src_sobelx, +void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, int width) { asm volatile("sub %0,%1 \n" - // 8 pixel loop. + // 32 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x00(%0,%1,1),%%xmm1 \n" - "lea 0x10(%0),%0 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x10,%3 \n" + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vpaddusb 0x00(%0,%1,1),%%ymm0,%%ymm0 \n" + "lea 0x20(%0),%0 \n" + "vmovdqu %%ymm0,(%2) \n" + "lea 0x20(%2),%2 \n" + "sub $0x20,%3 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_sobelx), // %0 "+r"(src_sobely), // %1 "+r"(dst_y), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1"); + : "memory", "cc", "xmm0"); } -#endif // HAS_SOBELTOPLANEROW_SSE2 +#endif // HAS_SOBELTOPLANEROW_AVX2 -#ifdef HAS_SOBELXYROW_SSE2 +#ifdef HAS_SOBELXYROW_AVX2 // Mixes Sobel X, Sobel Y and Sobel into ARGB. // A = 255 // R = Sobel X // G = Sobel // B = Sobel Y -void SobelXYRow_SSE2(const uint8_t* src_sobelx, +void SobelXYRow_AVX2(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width) { asm volatile( "sub %0,%1 \n" - "pcmpeqb %%xmm5,%%xmm5 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" - // 8 pixel loop. + // 16 pixel loop. LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x00(%0,%1,1),%%xmm1 \n" + "1: \n" + "vmovdqu (%0),%%xmm0 \n" + "vmovdqu 0x00(%0,%1,1),%%xmm1 \n" "lea 0x10(%0),%0 \n" - "movdqa %%xmm0,%%xmm2 \n" - "paddusb %%xmm1,%%xmm2 \n" - "movdqa %%xmm0,%%xmm3 \n" - "punpcklbw %%xmm5,%%xmm3 \n" - "punpckhbw %%xmm5,%%xmm0 \n" - "movdqa %%xmm1,%%xmm4 \n" - "punpcklbw %%xmm2,%%xmm4 \n" - "punpckhbw %%xmm2,%%xmm1 \n" - "movdqa %%xmm4,%%xmm6 \n" - "punpcklwd %%xmm3,%%xmm6 \n" - "punpckhwd %%xmm3,%%xmm4 \n" - "movdqa %%xmm1,%%xmm7 \n" - "punpcklwd %%xmm0,%%xmm7 \n" - "punpckhwd %%xmm0,%%xmm1 \n" - "movdqu %%xmm6,(%2) \n" - "movdqu %%xmm4,0x10(%2) \n" - "movdqu %%xmm7,0x20(%2) \n" - "movdqu %%xmm1,0x30(%2) \n" + "vpaddusb %%xmm1,%%xmm0,%%xmm2 \n" + "vpshufd $0xd8,%%xmm0,%%xmm0 \n" + "vpshufd $0xd8,%%xmm1,%%xmm1 \n" + "vpshufd $0xd8,%%xmm2,%%xmm2 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm2,%%ymm2 \n" + "vpunpcklbw %%ymm2,%%ymm1,%%ymm1 \n" + "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n" + "vpunpcklwd %%ymm0,%%ymm1,%%ymm2 \n" + "vpunpckhwd %%ymm0,%%ymm1,%%ymm0 \n" + "vmovdqu %%ymm2,(%2) \n" + "vmovdqu %%ymm0,0x20(%2) \n" "lea 0x40(%2),%2 \n" "sub $0x10,%3 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_sobelx), // %0 "+r"(src_sobely), // %1 "+r"(dst_argb), // %2 "+r"(width) // %3 : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm5"); } -#endif // HAS_SOBELXYROW_SSE2 - -#ifdef HAS_COMPUTECUMULATIVESUMROW_SSE2 -// Creates a table of cumulative sums where each value is a sum of all values -// above and to the left of the value, inclusive of the value. -void ComputeCumulativeSumRow_SSE2(const uint8_t* row, - int32_t* cumsum, - const int32_t* previous_cumsum, - int width) { - asm volatile( - "pxor %%xmm0,%%xmm0 \n" - "pxor %%xmm1,%%xmm1 \n" - "sub $0x4,%3 \n" - "jl 49f \n" - "test $0xf,%1 \n" - "jne 49f \n" - - // 4 pixel loop. - LABELALIGN - "40: \n" - "movdqu (%0),%%xmm2 \n" - "lea 0x10(%0),%0 \n" - "movdqa %%xmm2,%%xmm4 \n" - "punpcklbw %%xmm1,%%xmm2 \n" - "movdqa %%xmm2,%%xmm3 \n" - "punpcklwd %%xmm1,%%xmm2 \n" - "punpckhwd %%xmm1,%%xmm3 \n" - "punpckhbw %%xmm1,%%xmm4 \n" - "movdqa %%xmm4,%%xmm5 \n" - "punpcklwd %%xmm1,%%xmm4 \n" - "punpckhwd %%xmm1,%%xmm5 \n" - "paddd %%xmm2,%%xmm0 \n" - "movdqu (%2),%%xmm2 \n" - "paddd %%xmm0,%%xmm2 \n" - "paddd %%xmm3,%%xmm0 \n" - "movdqu 0x10(%2),%%xmm3 \n" - "paddd %%xmm0,%%xmm3 \n" - "paddd %%xmm4,%%xmm0 \n" - "movdqu 0x20(%2),%%xmm4 \n" - "paddd %%xmm0,%%xmm4 \n" - "paddd %%xmm5,%%xmm0 \n" - "movdqu 0x30(%2),%%xmm5 \n" - "lea 0x40(%2),%2 \n" - "paddd %%xmm0,%%xmm5 \n" - "movdqu %%xmm2,(%1) \n" - "movdqu %%xmm3,0x10(%1) \n" - "movdqu %%xmm4,0x20(%1) \n" - "movdqu %%xmm5,0x30(%1) \n" - "lea 0x40(%1),%1 \n" - "sub $0x4,%3 \n" - "jge 40b \n" - - "49: \n" - "add $0x3,%3 \n" - "jl 19f \n" - - // 1 pixel loop. - LABELALIGN - "10: \n" - "movd (%0),%%xmm2 \n" - "lea 0x4(%0),%0 \n" - "punpcklbw %%xmm1,%%xmm2 \n" - "punpcklwd %%xmm1,%%xmm2 \n" - "paddd %%xmm2,%%xmm0 \n" - "movdqu (%2),%%xmm2 \n" - "lea 0x10(%2),%2 \n" - "paddd %%xmm0,%%xmm2 \n" - "movdqu %%xmm2,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x1,%3 \n" - "jge 10b \n" - - "19: \n" - : "+r"(row), // %0 - "+r"(cumsum), // %1 - "+r"(previous_cumsum), // %2 - "+r"(width) // %3 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); -} -#endif // HAS_COMPUTECUMULATIVESUMROW_SSE2 - -#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 -void CumulativeSumToAverageRow_SSE2(const int32_t* topleft, - const int32_t* botleft, - int width, - int area, - uint8_t* dst, - int count) { - asm volatile( - "movd %5,%%xmm5 \n" - "cvtdq2ps %%xmm5,%%xmm5 \n" - "rcpss %%xmm5,%%xmm4 \n" - "pshufd $0x0,%%xmm4,%%xmm4 \n" - "sub $0x4,%3 \n" - "jl 49f \n" - "cmpl $0x80,%5 \n" - "ja 40f \n" - - "pshufd $0x0,%%xmm5,%%xmm5 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "psrld $0x10,%%xmm6 \n" - "cvtdq2ps %%xmm6,%%xmm6 \n" - "addps %%xmm6,%%xmm5 \n" - "mulps %%xmm4,%%xmm5 \n" - "cvtps2dq %%xmm5,%%xmm5 \n" - "packssdw %%xmm5,%%xmm5 \n" - - // 4 pixel small loop. - LABELALIGN - "4: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "movdqu 0x20(%0),%%xmm2 \n" - "movdqu 0x30(%0),%%xmm3 \n" - "psubd 0x00(%0,%4,4),%%xmm0 \n" - "psubd 0x10(%0,%4,4),%%xmm1 \n" - "psubd 0x20(%0,%4,4),%%xmm2 \n" - "psubd 0x30(%0,%4,4),%%xmm3 \n" - "lea 0x40(%0),%0 \n" - "psubd (%1),%%xmm0 \n" - "psubd 0x10(%1),%%xmm1 \n" - "psubd 0x20(%1),%%xmm2 \n" - "psubd 0x30(%1),%%xmm3 \n" - "paddd 0x00(%1,%4,4),%%xmm0 \n" - "paddd 0x10(%1,%4,4),%%xmm1 \n" - "paddd 0x20(%1,%4,4),%%xmm2 \n" - "paddd 0x30(%1,%4,4),%%xmm3 \n" - "lea 0x40(%1),%1 \n" - "packssdw %%xmm1,%%xmm0 \n" - "packssdw %%xmm3,%%xmm2 \n" - "pmulhuw %%xmm5,%%xmm0 \n" - "pmulhuw %%xmm5,%%xmm2 \n" - "packuswb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" - "jge 4b \n" - "jmp 49f \n" - - // 4 pixel loop - LABELALIGN - "40: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "movdqu 0x20(%0),%%xmm2 \n" - "movdqu 0x30(%0),%%xmm3 \n" - "psubd 0x00(%0,%4,4),%%xmm0 \n" - "psubd 0x10(%0,%4,4),%%xmm1 \n" - "psubd 0x20(%0,%4,4),%%xmm2 \n" - "psubd 0x30(%0,%4,4),%%xmm3 \n" - "lea 0x40(%0),%0 \n" - "psubd (%1),%%xmm0 \n" - "psubd 0x10(%1),%%xmm1 \n" - "psubd 0x20(%1),%%xmm2 \n" - "psubd 0x30(%1),%%xmm3 \n" - "paddd 0x00(%1,%4,4),%%xmm0 \n" - "paddd 0x10(%1,%4,4),%%xmm1 \n" - "paddd 0x20(%1,%4,4),%%xmm2 \n" - "paddd 0x30(%1,%4,4),%%xmm3 \n" - "lea 0x40(%1),%1 \n" - "cvtdq2ps %%xmm0,%%xmm0 \n" - "cvtdq2ps %%xmm1,%%xmm1 \n" - "mulps %%xmm4,%%xmm0 \n" - "mulps %%xmm4,%%xmm1 \n" - "cvtdq2ps %%xmm2,%%xmm2 \n" - "cvtdq2ps %%xmm3,%%xmm3 \n" - "mulps %%xmm4,%%xmm2 \n" - "mulps %%xmm4,%%xmm3 \n" - "cvtps2dq %%xmm0,%%xmm0 \n" - "cvtps2dq %%xmm1,%%xmm1 \n" - "cvtps2dq %%xmm2,%%xmm2 \n" - "cvtps2dq %%xmm3,%%xmm3 \n" - "packssdw %%xmm1,%%xmm0 \n" - "packssdw %%xmm3,%%xmm2 \n" - "packuswb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" - "jge 40b \n" - - "49: \n" - "add $0x3,%3 \n" - "jl 19f \n" - - // 1 pixel loop - LABELALIGN - "10: \n" - "movdqu (%0),%%xmm0 \n" - "psubd 0x00(%0,%4,4),%%xmm0 \n" - "lea 0x10(%0),%0 \n" - "psubd (%1),%%xmm0 \n" - "paddd 0x00(%1,%4,4),%%xmm0 \n" - "lea 0x10(%1),%1 \n" - "cvtdq2ps %%xmm0,%%xmm0 \n" - "mulps %%xmm4,%%xmm0 \n" - "cvtps2dq %%xmm0,%%xmm0 \n" - "packssdw %%xmm0,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movd %%xmm0,(%2) \n" - "lea 0x4(%2),%2 \n" - "sub $0x1,%3 \n" - "jge 10b \n" - "19: \n" - : "+r"(topleft), // %0 - "+r"(botleft), // %1 - "+r"(dst), // %2 - "+rm"(count) // %3 - : "r"((ptrdiff_t)(width)), // %4 - "rm"(area) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); -} -#endif // HAS_CUMULATIVESUMTOAVERAGEROW_SSE2 +#endif // HAS_SOBELXYROW_AVX2 #ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2 static const uvec8 kCumSumTranspose = {0, 4, 8, 12, 1, 5, 9, 13, @@ -10806,24 +12076,24 @@ void ComputeCumulativeSumRow_AVX2(const uint8_t* row, int width) { asm volatile( "vpxor %%xmm0,%%xmm0,%%xmm0 \n" - "vbroadcasti128 %4,%%ymm5 \n" + "vbroadcasti128 %4,%%ymm5 \n" "vmovdqu %5,%%ymm6 \n" "vmovdqu %6,%%ymm7 \n" - "vpcmpeqd %%ymm8,%%ymm8,%%ymm8 \n" - "vpsrlw $0xf,%%ymm8,%%ymm8 \n" + "vpcmpeqd %%ymm1,%%ymm1,%%ymm1 \n" + "vpsrlw $0xf,%%ymm1,%%ymm1 \n" "sub $0x4,%3 \n" "jl 49f \n" // 4 pixel loop. LABELALIGN "40: \n" - "vbroadcasti128 (%0),%%ymm2 \n" + "vbroadcasti128 (%0),%%ymm2 \n" "lea 0x10(%0),%0 \n" "vpshufb %%ymm5,%%ymm2,%%ymm2 \n" "vpmaddubsw %%ymm6,%%ymm2,%%ymm3 \n" "vpmaddubsw %%ymm7,%%ymm2,%%ymm4 \n" - "vpmaddwd %%ymm8,%%ymm3,%%ymm3 \n" - "vpmaddwd %%ymm8,%%ymm4,%%ymm4 \n" + "vpmaddwd %%ymm1,%%ymm3,%%ymm3 \n" + "vpmaddwd %%ymm1,%%ymm4,%%ymm4 \n" "vperm2i128 $0x11,%%ymm4,%%ymm4,%%ymm2 \n" "vpaddd (%2),%%ymm3,%%ymm3 \n" "vpaddd 0x20(%2),%%ymm4,%%ymm4 \n" @@ -10863,8 +12133,8 @@ void ComputeCumulativeSumRow_AVX2(const uint8_t* row, : "m"(kCumSumTranspose), // %4 "m"(kCumSumMaddLo), // %5 "m"(kCumSumMaddHi) // %6 - : "memory", "cc", "xmm0", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", - "xmm8"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); } #endif // HAS_COMPUTECUMULATIVESUMROW_AVX2 @@ -10880,7 +12150,7 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vcvtsi2ssl %5,%%xmm5,%%xmm5 \n" "vrcpss %%xmm5,%%xmm5,%%xmm4 \n" "vpshufd $0x0,%%xmm4,%%xmm4 \n" - "sub $0x4,%3 \n" + "subl $0x4,%3 \n" "jl 19f \n" "vinserti128 $0x1,%%xmm4,%%ymm4,%%ymm4 \n" "cmpl $0x80,%5 \n" @@ -10893,7 +12163,7 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vmulss %%xmm4,%%xmm6,%%xmm6 \n" "vcvtps2dq %%xmm6,%%xmm6 \n" "vpackssdw %%xmm6,%%xmm6,%%xmm6 \n" - "vpbroadcastw %%xmm6,%%ymm6 \n" + "vpbroadcastw %%xmm6,%%ymm6 \n" // 4 pixel small loop. LABELALIGN @@ -10910,12 +12180,12 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "lea 0x40(%1),%1 \n" "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" "vpmulhuw %%ymm6,%%ymm0,%%ymm0 \n" - "vextracti128 $0x1,%%ymm0,%%xmm1 \n" + "vextracti128 $0x1,%%ymm0,%%xmm1 \n" "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" "vpshufd $0xd8,%%xmm0,%%xmm0 \n" "vmovdqu %%xmm0,(%2) \n" "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" + "subl $0x4,%3 \n" "jge 4b \n" "vzeroupper \n" "jmp 19f \n" @@ -10940,17 +12210,17 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vcvtps2dq %%ymm0,%%ymm0 \n" "vcvtps2dq %%ymm1,%%ymm1 \n" "vpackssdw %%ymm1,%%ymm0,%%ymm0 \n" - "vextracti128 $0x1,%%ymm0,%%xmm1 \n" + "vextracti128 $0x1,%%ymm0,%%xmm1 \n" "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" "vpshufd $0xd8,%%xmm0,%%xmm0 \n" "vmovdqu %%xmm0,(%2) \n" "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" + "subl $0x4,%3 \n" "jge 10b \n" "vzeroupper \n" "19: \n" - "add $0x3,%3 \n" + "addl $0x3,%3 \n" "jl 99f \n" // 1 pixel loop. @@ -10970,106 +12240,96 @@ void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" "vmovd %%xmm0,(%2) \n" "lea 0x4(%2),%2 \n" - "sub $0x1,%3 \n" + "subl $0x1,%3 \n" "jge 20b \n" "99: \n" : "+r"(topleft), // %0 "+r"(botleft), // %1 "+r"(dst), // %2 - "+r"(count) // %3 + "+rm"(count) // %3 : "r"((ptrdiff_t)(width)), // %4 - "r"(area) // %5 + "rm"(area) // %5 : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5", "xmm6"); } #endif // HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 -#ifdef HAS_ARGBAFFINEROW_SSE2 +#ifdef HAS_ARGBAFFINEROW_AVX2 +static const float kAffineRamp[8] = {0.0f, 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, 7.0f}; +static const float kAffineStep8 = 8.0f; // Copy ARGB pixels from source image with slope to a row of destination. LIBYUV_API -void ARGBAffineRow_SSE2(const uint8_t* src_argb, +void ARGBAffineRow_AVX2(const uint8_t* src_argb, int src_argb_stride, uint8_t* dst_argb, const float* src_dudv, int width) { - ptrdiff_t src_argb_stride_temp = src_argb_stride; - intptr_t temp; asm volatile( - "movq (%3),%%xmm2 \n" - "movq 0x08(%3),%%xmm7 \n" - "shl $0x10,%1 \n" - "add $0x4,%1 \n" - "movd %1,%%xmm5 \n" - "sub $0x4,%4 \n" - "jl 49f \n" + "vbroadcastss (%[uv]),%%ymm2 \n" + "vbroadcastss 0x04(%[uv]),%%ymm3 \n" + "vbroadcastss 0x08(%[uv]),%%ymm4 \n" + "vbroadcastss 0x0c(%[uv]),%%ymm5 \n" + "vmovups %[ramp],%%ymm0 \n" + "vmulps %%ymm0,%%ymm4,%%ymm1 \n" + "vaddps %%ymm1,%%ymm2,%%ymm2 \n" + "vmulps %%ymm0,%%ymm5,%%ymm1 \n" + "vaddps %%ymm1,%%ymm3,%%ymm3 \n" + "vbroadcastss %[step8],%%ymm0 \n" + "vmulps %%ymm0,%%ymm4,%%ymm4 \n" + "vmulps %%ymm0,%%ymm5,%%ymm5 \n" + "vpbroadcastd %[stride],%%ymm6 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" - "pshufd $0x44,%%xmm7,%%xmm7 \n" - "pshufd $0x0,%%xmm5,%%xmm5 \n" - "movdqa %%xmm2,%%xmm0 \n" - "addps %%xmm7,%%xmm0 \n" - "movlhps %%xmm0,%%xmm2 \n" - "movdqa %%xmm7,%%xmm4 \n" - "addps %%xmm4,%%xmm4 \n" - "movdqa %%xmm2,%%xmm3 \n" - "addps %%xmm4,%%xmm3 \n" - "addps %%xmm4,%%xmm4 \n" - - // 4 pixel loop LABELALIGN - "40: \n" - "cvttps2dq %%xmm2,%%xmm0 \n" // x,y float->int first 2 - "cvttps2dq %%xmm3,%%xmm1 \n" // x,y float->int next 2 - "packssdw %%xmm1,%%xmm0 \n" // x, y as 8 shorts - "pmaddwd %%xmm5,%%xmm0 \n" // off = x*4 + y*stride - "movd %%xmm0,%k1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" - "movd %%xmm0,%k5 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" - "movd 0x00(%0,%1,1),%%xmm1 \n" - "movd 0x00(%0,%5,1),%%xmm6 \n" - "punpckldq %%xmm6,%%xmm1 \n" - "addps %%xmm4,%%xmm2 \n" - "movq %%xmm1,(%2) \n" - "movd %%xmm0,%k1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" - "movd %%xmm0,%k5 \n" - "movd 0x00(%0,%1,1),%%xmm0 \n" - "movd 0x00(%0,%5,1),%%xmm6 \n" - "punpckldq %%xmm6,%%xmm0 \n" - "addps %%xmm4,%%xmm3 \n" - "movq %%xmm0,0x08(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%4 \n" - "jge 40b \n" + "1: \n" + "vcvttps2dq %%ymm2,%%ymm0 \n" + "vcvttps2dq %%ymm3,%%ymm1 \n" + "vpslld $2,%%ymm0,%%ymm0 \n" + "vpmulld %%ymm6,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[src],%%ymm0,1),%%ymm1\n" + "vaddps %%ymm4,%%ymm2,%%ymm2 \n" + "vaddps %%ymm5,%%ymm3,%%ymm3 \n" + "vmovdqu %%ymm1,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" - "49: \n" - "add $0x3,%4 \n" - "jl 19f \n" + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + "vmovss 0x08(%[uv]),%%xmm4 \n" + "vmovss 0x0c(%[uv]),%%xmm5 \n" - // 1 pixel loop + // 1 pixel remainder loop. LABELALIGN - "10: \n" - "cvttps2dq %%xmm2,%%xmm0 \n" - "packssdw %%xmm0,%%xmm0 \n" - "pmaddwd %%xmm5,%%xmm0 \n" - "addps %%xmm7,%%xmm2 \n" - "movd %%xmm0,%k1 \n" - "movd 0x00(%0,%1,1),%%xmm0 \n" - "movd %%xmm0,(%2) \n" - "lea 0x04(%2),%2 \n" - "sub $0x1,%4 \n" - "jge 10b \n" - "19: \n" - : "+r"(src_argb), // %0 - "+r"(src_argb_stride_temp), // %1 - "+r"(dst_argb), // %2 - "+r"(src_dudv), // %3 - "+rm"(width), // %4 - "=&r"(temp) // %5 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + "3: \n" + "vcvttps2dq %%xmm2,%%xmm0 \n" + "vcvttps2dq %%xmm3,%%xmm1 \n" + "vpslld $2,%%xmm0,%%xmm0 \n" + "vpmulld %%xmm6,%%xmm1,%%xmm1 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpcmpeqd %%xmm7,%%xmm7,%%xmm7 \n" + "vpsrldq $12,%%xmm7,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[src],%%xmm0,1),%%xmm1\n" + "vaddss %%xmm4,%%xmm2,%%xmm2 \n" + "vaddss %%xmm5,%%xmm3,%%xmm3 \n" + "vmovd %%xmm1,(%[dst]) \n" + "lea 0x04(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [width] "+r"(width) + : [src] "r"(src_argb), [uv] "r"(src_dudv), [stride] "m"(src_argb_stride), + [ramp] "m"(kAffineRamp), [step8] "m"(kAffineStep8) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); } -#endif // HAS_ARGBAFFINEROW_SSE2 +#endif // HAS_ARGBAFFINEROW_AVX2 #ifdef HAS_INTERPOLATEROW_AVX2 // Bilinear filter 32x2 -> 32x1 @@ -11701,11 +12961,14 @@ void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, } #endif // HAS_ARGBPOLYNOMIALROW_AVX2 -#ifdef HAS_HALFFLOATROW_AVX2 +#if defined(HAS_HALFFLOATROW_AVX2) || defined(HAS_HALFFLOATROW_AVX512BW) // 2^-112 moves the float exponent into half float position. Applied as a // separate multiply after scale so no denormal constant is formed when // scale < 2^-14 (see HalfFloatRow_C). static const float kScaleBias = 1.9259299444e-34f; +#endif + +#ifdef HAS_HALFFLOATROW_AVX2 void HalfFloatRow_AVX2(const uint16_t* src, uint16_t* dst, float scale, @@ -11751,6 +13014,54 @@ void HalfFloatRow_AVX2(const uint16_t* src, } #endif // HAS_HALFFLOATROW_AVX2 +#ifdef HAS_HALFFLOATROW_AVX512BW +void HalfFloatRow_AVX512BW(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + uintptr_t temp; + asm volatile( + "vbroadcastss %[scale],%%zmm4 \n" + "vbroadcastss %[bias],%%zmm5 \n" + "sub $0x10,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vpmovzxwd (%[src]),%%zmm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vcvtdq2ps %%zmm0,%%zmm0 \n" + "vmulps %%zmm4,%%zmm0,%%zmm0 \n" + "vmulps %%zmm5,%%zmm0,%%zmm0 \n" + "vpsrld $0xd,%%zmm0,%%zmm0 \n" + "vpmovdw %%zmm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k1 \n" + "vpmovzxwd (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vcvtdq2ps %%zmm0,%%zmm0 \n" + "vmulps %%zmm4,%%zmm0,%%zmm0 \n" + "vmulps %%zmm5,%%zmm0,%%zmm0 \n" + "vpsrld $0xd,%%zmm0,%%zmm0 \n" + "vpmovdw %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : + [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width), [temp] "=&r"(temp) + : [scale] "m"(scale), [bias] "m"(kScaleBias) + : "memory", "cc", "xmm0", "xmm4", "xmm5", "k1"); +} +#endif // HAS_HALFFLOATROW_AVX512BW + #ifdef HAS_HALFFLOATROW_F16C void HalfFloatRow_F16C(const uint16_t* src, uint16_t* dst, @@ -11816,169 +13127,974 @@ void HalfFloat1Row_F16C(const uint16_t* src, uint16_t* dst, float, int width) { } #endif // HAS_HALFFLOATROW_F16C -#ifdef HAS_ARGBCOLORTABLEROW_X86 -// Tranform ARGB pixels with color table. -void ARGBColorTableRow_X86(uint8_t* dst_argb, +#ifdef HAS_BYTETOFLOATROW_AVX2 +void ByteToFloatRow_AVX2(const uint8_t* src, + float* dst, + float scale, + int width) { + asm volatile("vbroadcastss %3,%%ymm2 \n" + + LABELALIGN + "1: \n" + "vpmovzxbd (%0),%%ymm0 \n" + "vpmovzxbd 0x08(%0),%%ymm1 \n" + "lea 0x10(%0),%0 \n" + "vcvtdq2ps %%ymm0,%%ymm0 \n" + "vcvtdq2ps %%ymm1,%%ymm1 \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmulps %%ymm2,%%ymm1,%%ymm1 \n" + "vmovups %%ymm0,(%1) \n" + "vmovups %%ymm1,0x20(%1) \n" + "lea 0x40(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src), // %0 + "+r"(dst), // %1 + "+r"(width) // %2 +#if defined(__x86_64__) + : "x"(scale) // %3 +#else + : "m"(scale) // %3 +#endif + : "memory", "cc", "xmm0", "xmm1", "xmm2"); +} +#endif // HAS_BYTETOFLOATROW_AVX2 + +#ifdef HAS_GAUSSROW_AVX2 +static const int32_t kGaussRound128 = 128; +static const int8_t kShufGauss32To16[16] = {0, 1, 4, 5, 8, 9, 12, 13, + -1, -1, -1, -1, -1, -1, -1, -1}; +void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width) { + asm volatile( + "vpbroadcastd %[k128],%%ymm4 \n" + "vbroadcasti128 %[kShuf],%%ymm5 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vmovdqu 0x04(%[src]),%%ymm1 \n" + "vmovdqu 0x08(%[src]),%%ymm2 \n" + "vpaddd 0x10(%[src]),%%ymm0,%%ymm0 \n" + "vpaddd 0x0c(%[src]),%%ymm1,%%ymm1 \n" + "lea 0x20(%[src]),%[src] \n" + "vpslld $2,%%ymm1,%%ymm1 \n" + "vpslld $1,%%ymm2,%%ymm3 \n" + "vpslld $2,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm4,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vpsrld $8,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm5,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovd (%[src]),%%xmm0 \n" + "vmovd 0x04(%[src]),%%xmm1 \n" + "vmovd 0x08(%[src]),%%xmm2 \n" + "vmovd 0x0c(%[src]),%%xmm3 \n" + "vpaddd %%xmm3,%%xmm1,%%xmm1 \n" + "vmovd 0x10(%[src]),%%xmm3 \n" + "vpaddd %%xmm3,%%xmm0,%%xmm0 \n" + "lea 0x04(%[src]),%[src] \n" + "vpslld $2,%%xmm1,%%xmm1 \n" + "vpslld $1,%%xmm2,%%xmm3 \n" + "vpslld $2,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm3,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm4,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" + "vpsrld $8,%%xmm0,%%xmm0 \n" + "vpextrw $0,%%xmm0,(%[dst]) \n" + "lea 0x02(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) + : [k128] "m"(kGaussRound128), [kShuf] "m"(kShufGauss32To16) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +} +#endif // HAS_GAUSSROW_AVX2 + +#ifdef HAS_GAUSSCOL_AVX2 +void GaussCol_AVX2(const uint16_t* src0, + const uint16_t* src1, + const uint16_t* src2, + const uint16_t* src3, + const uint16_t* src4, + uint32_t* dst, + int width) { +#if defined(__x86_64__) + ptrdiff_t off1 = (ptrdiff_t)((uintptr_t)src1 - (uintptr_t)src0); + ptrdiff_t off2 = (ptrdiff_t)((uintptr_t)src2 - (uintptr_t)src0); + ptrdiff_t off3 = (ptrdiff_t)((uintptr_t)src3 - (uintptr_t)src0); + ptrdiff_t off4 = (ptrdiff_t)((uintptr_t)src4 - (uintptr_t)src0); + asm volatile( + "vpxor %%xmm5,%%xmm5,%%xmm5 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vpmovzxwd (%[p0]),%%ymm0 \n" + "vpmovzxwd (%[p0],%[off4],1),%%ymm4 \n" + "vpmovzxwd (%[p0],%[off1],1),%%ymm1 \n" + "vpmovzxwd (%[p0],%[off3],1),%%ymm3 \n" + "vpmovzxwd (%[p0],%[off2],1),%%ymm2 \n" + "lea 0x10(%[p0]),%[p0] \n" + "vpaddd %%ymm4,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm3,%%ymm1,%%ymm1 \n" + "vpslld $2,%%ymm1,%%ymm1 \n" + "vpslld $1,%%ymm2,%%ymm3 \n" + "vpslld $2,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dp]) \n" + "lea 0x20(%[dp]),%[dp] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vpinsrw $0,(%[p0]),%%xmm5,%%xmm0 \n" + "vpinsrw $0,(%[p0],%[off4],1),%%xmm5,%%xmm4\n" + "vpinsrw $0,(%[p0],%[off1],1),%%xmm5,%%xmm1\n" + "vpinsrw $0,(%[p0],%[off3],1),%%xmm5,%%xmm3\n" + "vpinsrw $0,(%[p0],%[off2],1),%%xmm5,%%xmm2\n" + "lea 0x02(%[p0]),%[p0] \n" + "vpaddd %%xmm4,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm3,%%xmm1,%%xmm1 \n" + "vpslld $2,%%xmm1,%%xmm1 \n" + "vpslld $1,%%xmm2,%%xmm3 \n" + "vpslld $2,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm3,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%[dp]) \n" + "lea 0x04(%[dp]),%[dp] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [p0] "+r"(src0), [dp] "+r"(dst), [width] "+r"(width) + : [off1] "r"(off1), [off2] "r"(off2), [off3] "r"(off3), + [off4] "r"(off4) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#else + int n = width & ~7; + ptrdiff_t idx = 0; + const uint16_t* tmp; + asm volatile( + "vpxor %%xmm5,%%xmm5,%%xmm5 \n" + "cmpl $0,%[n] \n" + "jle 2f \n" + + LABELALIGN + "1: \n" + "mov %[s0],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm0 \n" + "mov %[s4],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm4 \n" + "mov %[s1],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm1 \n" + "mov %[s3],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm3 \n" + "mov %[s2],%[tmp] \n" + "vpmovzxwd (%[tmp],%[idx],2),%%ymm2 \n" + "add $0x8,%[idx] \n" + "vpaddd %%ymm4,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm3,%%ymm1,%%ymm1 \n" + "vpslld $2,%%ymm1,%%ymm1 \n" + "vpslld $1,%%ymm2,%%ymm3 \n" + "vpslld $2,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm1,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dp]) \n" + "lea 0x20(%[dp]),%[dp] \n" + "cmpl %[n],%[idx] \n" + "jl 1b \n" + + "2: \n" + "cmpl %[width],%[idx] \n" + "jge 99f \n" + + LABELALIGN + "3: \n" + "mov %[s0],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm0\n" + "mov %[s4],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm4\n" + "mov %[s1],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm1\n" + "mov %[s3],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm3\n" + "mov %[s2],%[tmp] \n" + "vpinsrw $0,(%[tmp],%[idx],2),%%xmm5,%%xmm2\n" + "add $0x1,%[idx] \n" + "vpaddd %%xmm4,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm3,%%xmm1,%%xmm1 \n" + "vpslld $2,%%xmm1,%%xmm1 \n" + "vpslld $1,%%xmm2,%%xmm3 \n" + "vpslld $2,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm3,%%xmm2,%%xmm2 \n" + "vpaddd %%xmm1,%%xmm0,%%xmm0 \n" + "vpaddd %%xmm2,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,(%[dp]) \n" + "lea 0x04(%[dp]),%[dp] \n" + "cmpl %[width],%[idx] \n" + "jl 3b \n" + + "99: \n" + "vzeroupper \n" + : [idx] "+r"(idx), [tmp] "=&r"(tmp), [dp] "+r"(dst) + : [s0] "m"(src0), [s1] "m"(src1), [s2] "m"(src2), [s3] "m"(src3), + [s4] "m"(src4), [n] "m"(n), [width] "m"(width) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif +} +#endif // HAS_GAUSSCOL_AVX2 + +#if defined(HAS_GAUSSROW_F32_AVX2) || defined(HAS_GAUSSCOL_F32_AVX2) +static const float kGaussF32_4 = 4.0f; +static const float kGaussF32_6 = 6.0f; +#endif // defined(HAS_GAUSSROW_F32_AVX2) || defined(HAS_GAUSSCOL_F32_AVX2) + +#ifdef HAS_GAUSSROW_F32_AVX2 +static const float kGaussF32_Scale = 1.0f / 256.0f; +void GaussRow_F32_AVX2(const float* src, float* dst, int width) { + asm volatile( + "vbroadcastss %[k4],%%ymm4 \n" + "vbroadcastss %[k6],%%ymm5 \n" + "vbroadcastss %[kScale],%%ymm6 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "vmulps 0x04(%[src]),%%ymm4,%%ymm1 \n" + "vmulps 0x08(%[src]),%%ymm5,%%ymm2 \n" + "vmulps 0x0c(%[src]),%%ymm4,%%ymm3 \n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" + "vaddps %%ymm2,%%ymm0,%%ymm0 \n" + "vaddps %%ymm3,%%ymm0,%%ymm0 \n" + "vaddps 0x10(%[src]),%%ymm0,%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmulps %%ymm6,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "vmulss 0x04(%[src]),%%xmm4,%%xmm1 \n" + "vmulss 0x08(%[src]),%%xmm5,%%xmm2 \n" + "vmulss 0x0c(%[src]),%%xmm4,%%xmm3 \n" + "vaddss %%xmm1,%%xmm0,%%xmm0 \n" + "vaddss %%xmm2,%%xmm0,%%xmm0 \n" + "vaddss %%xmm3,%%xmm0,%%xmm0 \n" + "vaddss 0x10(%[src]),%%xmm0,%%xmm0 \n" + "lea 0x04(%[src]),%[src] \n" + "vmulss %%xmm6,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x04(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) + : [k4] "m"(kGaussF32_4), [k6] "m"(kGaussF32_6), + [kScale] "m"(kGaussF32_Scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6"); +} +#endif // HAS_GAUSSROW_F32_AVX2 + +#ifdef HAS_GAUSSCOL_F32_AVX2 +void GaussCol_F32_AVX2(const float* src0, + const float* src1, + const float* src2, + const float* src3, + const float* src4, + float* dst, + int width) { +#if defined(__x86_64__) + ptrdiff_t off1 = (ptrdiff_t)((uintptr_t)src1 - (uintptr_t)src0); + ptrdiff_t off2 = (ptrdiff_t)((uintptr_t)src2 - (uintptr_t)src0); + ptrdiff_t off3 = (ptrdiff_t)((uintptr_t)src3 - (uintptr_t)src0); + ptrdiff_t off4 = (ptrdiff_t)((uintptr_t)src4 - (uintptr_t)src0); + asm volatile( + "vbroadcastss %[k4],%%ymm4 \n" + "vbroadcastss %[k6],%%ymm5 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[p0]),%%ymm0 \n" + "vmulps (%[p0],%[off1],1),%%ymm4,%%ymm1\n" + "vmulps (%[p0],%[off2],1),%%ymm5,%%ymm2\n" + "vmulps (%[p0],%[off3],1),%%ymm4,%%ymm3\n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" + "vaddps %%ymm2,%%ymm0,%%ymm0 \n" + "vaddps %%ymm3,%%ymm0,%%ymm0 \n" + "vaddps (%[p0],%[off4],1),%%ymm0,%%ymm0\n" + "lea 0x20(%[p0]),%[p0] \n" + "vmovups %%ymm0,(%[dp]) \n" + "lea 0x20(%[dp]),%[dp] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[p0]),%%xmm0 \n" + "vmulss (%[p0],%[off1],1),%%xmm4,%%xmm1\n" + "vmulss (%[p0],%[off2],1),%%xmm5,%%xmm2\n" + "vmulss (%[p0],%[off3],1),%%xmm4,%%xmm3\n" + "vaddss %%xmm1,%%xmm0,%%xmm0 \n" + "vaddss %%xmm2,%%xmm0,%%xmm0 \n" + "vaddss %%xmm3,%%xmm0,%%xmm0 \n" + "vaddss (%[p0],%[off4],1),%%xmm0,%%xmm0\n" + "lea 0x04(%[p0]),%[p0] \n" + "vmovss %%xmm0,(%[dp]) \n" + "lea 0x04(%[dp]),%[dp] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [p0] "+r"(src0), [dp] "+r"(dst), [width] "+r"(width) + : [off1] "r"(off1), [off2] "r"(off2), [off3] "r"(off3), + [off4] "r"(off4), [k4] "m"(kGaussF32_4), [k6] "m"(kGaussF32_6) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#else + int n = width & ~7; + ptrdiff_t idx = 0; + const float* tmp; + asm volatile( + "vbroadcastss %[k4],%%ymm4 \n" + "vbroadcastss %[k6],%%ymm5 \n" + "cmpl $0,%[n] \n" + "jle 2f \n" + + LABELALIGN + "1: \n" + "mov %[s0],%[tmp] \n" + "vmovups (%[tmp],%[idx],4),%%ymm0 \n" + "mov %[s1],%[tmp] \n" + "vmulps (%[tmp],%[idx],4),%%ymm4,%%ymm1\n" + "mov %[s2],%[tmp] \n" + "vmulps (%[tmp],%[idx],4),%%ymm5,%%ymm2\n" + "mov %[s3],%[tmp] \n" + "vmulps (%[tmp],%[idx],4),%%ymm4,%%ymm3\n" + "vaddps %%ymm1,%%ymm0,%%ymm0 \n" + "vaddps %%ymm2,%%ymm0,%%ymm0 \n" + "vaddps %%ymm3,%%ymm0,%%ymm0 \n" + "mov %[s4],%[tmp] \n" + "vaddps (%[tmp],%[idx],4),%%ymm0,%%ymm0\n" + "add $0x8,%[idx] \n" + "vmovups %%ymm0,(%[dp]) \n" + "lea 0x20(%[dp]),%[dp] \n" + "cmpl %[n],%[idx] \n" + "jl 1b \n" + + "2: \n" + "cmpl %[width],%[idx] \n" + "jge 99f \n" + + LABELALIGN + "3: \n" + "mov %[s0],%[tmp] \n" + "vmovss (%[tmp],%[idx],4),%%xmm0 \n" + "mov %[s1],%[tmp] \n" + "vmulss (%[tmp],%[idx],4),%%xmm4,%%xmm1\n" + "mov %[s2],%[tmp] \n" + "vmulss (%[tmp],%[idx],4),%%xmm5,%%xmm2\n" + "mov %[s3],%[tmp] \n" + "vmulss (%[tmp],%[idx],4),%%xmm4,%%xmm3\n" + "vaddss %%xmm1,%%xmm0,%%xmm0 \n" + "vaddss %%xmm2,%%xmm0,%%xmm0 \n" + "vaddss %%xmm3,%%xmm0,%%xmm0 \n" + "mov %[s4],%[tmp] \n" + "vaddss (%[tmp],%[idx],4),%%xmm0,%%xmm0\n" + "add $0x1,%[idx] \n" + "vmovss %%xmm0,(%[dp]) \n" + "lea 0x04(%[dp]),%[dp] \n" + "cmpl %[width],%[idx] \n" + "jl 3b \n" + + "99: \n" + "vzeroupper \n" + : [idx] "+r"(idx), [tmp] "=&r"(tmp), [dp] "+r"(dst) + : [s0] "m"(src0), [s1] "m"(src1), [s2] "m"(src2), [s3] "m"(src3), + [s4] "m"(src4), [n] "m"(n), [width] "m"(width), [k4] "m"(kGaussF32_4), + [k6] "m"(kGaussF32_6) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +#endif +} +#endif // HAS_GAUSSCOL_F32_AVX2 + +#ifdef HAS_SCALEMAXSAMPLES_AVX2 +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + float fmax; + asm volatile( + "vpxor %%xmm3,%%xmm3,%%xmm3 \n" + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmaxps %%ymm3,%%ymm0,%%ymm3 \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "vextractf128 $0x1,%%ymm3,%%xmm1 \n" + "vmaxps %%xmm3,%%xmm1,%%xmm3 \n" + "vpshufd $0x4e,%%xmm3,%%xmm1 \n" + "vmaxps %%xmm3,%%xmm1,%%xmm3 \n" + "vpshufd $0x55,%%xmm3,%%xmm1 \n" + "vmaxss %%xmm3,%%xmm1,%%xmm3 \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmaxss %%xmm3,%%xmm0,%%xmm3 \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vmovss %%xmm3,%[fmax] \n" + "vzeroupper \n" + : [src] "+r"(src), + [dst] "+r"(dst), + [width] "+r"(width), + [fmax] "=m"(fmax) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_AVX2 + +#ifdef HAS_SCALESUMSAMPLES_AVX2 +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + float fsum; + asm volatile( + "vpxor %%xmm3,%%xmm3,%%xmm3 \n" + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmulps %%ymm0,%%ymm0,%%ymm1 \n" + "vaddps %%ymm1,%%ymm3,%%ymm3 \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "vextractf128 $0x1,%%ymm3,%%xmm1 \n" + "vaddps %%xmm1,%%xmm3,%%xmm3 \n" + "vhaddps %%xmm3,%%xmm3,%%xmm3 \n" + "vhaddps %%xmm3,%%xmm3,%%xmm3 \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmulss %%xmm0,%%xmm0,%%xmm1 \n" + "vaddss %%xmm1,%%xmm3,%%xmm3 \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vmovss %%xmm3,%[fsum] \n" + "vzeroupper \n" + : [src] "+r"(src), + [dst] "+r"(dst), + [width] "+r"(width), + [fsum] "=m"(fsum) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_AVX2 + +#ifdef HAS_SCALESAMPLES_AVX2 +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { + asm volatile( + "vbroadcastss %[scale],%%ymm2 \n" + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovups (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vmulps %%ymm2,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovss (%[src]),%%xmm0 \n" + "lea 0x4(%[src]),%[src] \n" + "vmulss %%xmm2,%%xmm0,%%xmm0 \n" + "vmovss %%xmm0,(%[dst]) \n" + "lea 0x4(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src), [dst] "+r"(dst), [width] "+r"(width) + : [scale] "m"(scale) + : "memory", "cc", "xmm0", "xmm2"); +} +#endif // HAS_SCALESAMPLES_AVX2 + +#ifdef HAS_ARGBCOLORTABLEROW_AVX2 +// Transform ARGB pixels with color table. +void ARGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + asm volatile( + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrld $24,%%ymm6,%%ymm6 \n" // 0x000000ff + "sub $0x8,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[dst]),%%ymm0 \n" + "vpand %%ymm6,%%ymm0,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm2\n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + + "vpsrld $8,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $8,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vpsrld $16,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $16,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vpsrld $24,%%ymm0,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $24,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vmovdqu %%ymm2,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + "vpcmpeqd %%xmm5,%%xmm5,%%xmm5 \n" + "vpsrldq $12,%%xmm5,%%xmm5 \n" + + LABELALIGN + "3: \n" + "vmovd (%[dst]),%%xmm0 \n" + "vpand %%xmm6,%%xmm0,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm2\n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + + "vpsrld $8,%%xmm0,%%xmm1 \n" + "vpand %%xmm6,%%xmm1,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $8,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vpsrld $16,%%xmm0,%%xmm1 \n" + "vpand %%xmm6,%%xmm1,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $16,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vpsrld $24,%%xmm0,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $24,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vmovd %%xmm2,(%[dst]) \n" + "lea 0x04(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [width] "+r"(width) + : [table] "r"(table_argb) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#endif // HAS_ARGBCOLORTABLEROW_AVX2 + +#ifdef HAS_RGBCOLORTABLEROW_AVX2 +// Transform RGB pixels with color table. +void RGBColorTableRow_AVX2(uint8_t* dst_argb, const uint8_t* table_argb, int width) { - uintptr_t pixel_temp; asm volatile( - // 1 pixel loop. - LABELALIGN - "1: \n" - "movzb (%0),%1 \n" - "lea 0x4(%0),%0 \n" - "movzb 0x00(%3,%1,4),%1 \n" - "mov %b1,-0x4(%0) \n" - "movzb -0x3(%0),%1 \n" - "movzb 0x01(%3,%1,4),%1 \n" - "mov %b1,-0x3(%0) \n" - "movzb -0x2(%0),%1 \n" - "movzb 0x02(%3,%1,4),%1 \n" - "mov %b1,-0x2(%0) \n" - "movzb -0x1(%0),%1 \n" - "movzb 0x03(%3,%1,4),%1 \n" - "mov %b1,-0x1(%0) \n" - "dec %2 \n" - "jg 1b \n" - : "+r"(dst_argb), // %0 - "=&d"(pixel_temp), // %1 - "+r"(width) // %2 - : "r"(table_argb) // %3 - : "memory", "cc"); -} -#endif // HAS_ARGBCOLORTABLEROW_X86 + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrld $24,%%ymm6,%%ymm6 \n" // 0x000000ff + "sub $0x8,%[width] \n" + "jl 2f \n" -#ifdef HAS_RGBCOLORTABLEROW_X86 -// Tranform RGB pixels with color table. -void RGBColorTableRow_X86(uint8_t* dst_argb, - const uint8_t* table_argb, - int width) { - uintptr_t pixel_temp; - asm volatile( - // 1 pixel loop. LABELALIGN - "1: \n" - "movzb (%0),%1 \n" - "lea 0x4(%0),%0 \n" - "movzb 0x00(%3,%1,4),%1 \n" - "mov %b1,-0x4(%0) \n" - "movzb -0x3(%0),%1 \n" - "movzb 0x01(%3,%1,4),%1 \n" - "mov %b1,-0x3(%0) \n" - "movzb -0x2(%0),%1 \n" - "movzb 0x02(%3,%1,4),%1 \n" - "mov %b1,-0x2(%0) \n" - "dec %2 \n" - "jg 1b \n" - : "+r"(dst_argb), // %0 - "=&d"(pixel_temp), // %1 - "+r"(width) // %2 - : "r"(table_argb) // %3 - : "memory", "cc"); -} -#endif // HAS_RGBCOLORTABLEROW_X86 + "1: \n" + "vmovdqu (%[dst]),%%ymm0 \n" + "vpand %%ymm6,%%ymm0,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm2\n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" -#ifdef HAS_ARGBLUMACOLORTABLEROW_SSSE3 -// Tranform RGB pixels with luma table. -void ARGBLumaColorTableRow_SSSE3(const uint8_t* src_argb, - uint8_t* dst_argb, - int width, - const uint8_t* luma, - uint32_t lumacoeff) { + "vpsrld $8,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $8,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vpsrld $16,%%ymm0,%%ymm1 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdd %%ymm7,0(%[table],%%ymm1,4),%%ymm3\n" + "vpslld $16,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + + "vpslld $24,%%ymm6,%%ymm4 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpor %%ymm0,%%ymm2,%%ymm2 \n" + + "vmovdqu %%ymm2,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x8,%[width] \n" + "jle 99f \n" + "vpcmpeqd %%xmm5,%%xmm5,%%xmm5 \n" + "vpsrldq $12,%%xmm5,%%xmm5 \n" + + LABELALIGN + "3: \n" + "vmovd (%[dst]),%%xmm0 \n" + "vpand %%xmm6,%%xmm0,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm2\n" + "vpand %%xmm6,%%xmm2,%%xmm2 \n" + + "vpsrld $8,%%xmm0,%%xmm1 \n" + "vpand %%xmm6,%%xmm1,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $8,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vpsrld $16,%%xmm0,%%xmm1 \n" + "vpand %%xmm6,%%xmm1,%%xmm1 \n" + "vmovdqa %%xmm5,%%xmm7 \n" + "vpgatherdd %%xmm7,0(%[table],%%xmm1,4),%%xmm3\n" + "vpslld $16,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm3,%%xmm3 \n" + "vpor %%xmm3,%%xmm2,%%xmm2 \n" + + "vpslld $24,%%xmm6,%%xmm4 \n" + "vpand %%xmm4,%%xmm0,%%xmm0 \n" + "vpor %%xmm0,%%xmm2,%%xmm2 \n" + + "vmovd %%xmm2,(%[dst]) \n" + "lea 0x04(%[dst]),%[dst] \n" + "sub $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [width] "+r"(width) + : [table] "r"(table_argb) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); +} +#endif // HAS_RGBCOLORTABLEROW_AVX2 + +#ifdef HAS_ARGBLUMACOLORTABLEROW_AVX2 +// Transform RGB pixels with luma table. +void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff) { + uint32_t coeff = lumacoeff & 0x00ffffffu; uintptr_t pixel_temp; uintptr_t table_temp; asm volatile( - "movd %6,%%xmm3 \n" - "pshufd $0x0,%%xmm3,%%xmm3 \n" - "pcmpeqb %%xmm4,%%xmm4 \n" - "psllw $0x8,%%xmm4 \n" - "pxor %%xmm5,%%xmm5 \n" + "vpbroadcastd %[coeff],%%ymm5 \n" + "vpcmpeqd %%xmm4,%%xmm4,%%xmm4 \n" + "vpsrlw $9,%%xmm4,%%xmm4 \n" + "vpsllw $8,%%xmm4,%%xmm4 \n" // 0x7f00 + "subl $0x8,%[width] \n" + "jl 2f \n" - // 4 pixel loop. LABELALIGN - "1: \n" - "movdqu (%2),%%xmm0 \n" - "pmaddubsw %%xmm3,%%xmm0 \n" - "phaddw %%xmm0,%%xmm0 \n" - "pand %%xmm4,%%xmm0 \n" - "punpcklwd %%xmm5,%%xmm0 \n" - "movd %%xmm0,%k1 \n" // 32 bit offset - "add %5,%1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vpmaddubsw %%ymm5,%%ymm0,%%ymm1 \n" + "vphaddw %%ymm1,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpand %%xmm4,%%xmm1,%%xmm1 \n" - "movzb (%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,(%3) \n" - "movzb 0x1(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x1(%3) \n" - "movzb 0x2(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x2(%3) \n" - "movzb 0x3(%2),%0 \n" - "mov %b0,0x3(%3) \n" + "vpextrw $0,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x00(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x00(%[dst]) \n" + "movzbl 0x01(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x01(%[dst]) \n" + "movzbl 0x02(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x02(%[dst]) \n" + "movzbl 0x03(%[src]),%k[t0] \n" + "mov %b[t0],0x03(%[dst]) \n" - "movd %%xmm0,%k1 \n" // 32 bit offset - "add %5,%1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" + "vpextrw $1,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x04(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x04(%[dst]) \n" + "movzbl 0x05(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x05(%[dst]) \n" + "movzbl 0x06(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x06(%[dst]) \n" + "movzbl 0x07(%[src]),%k[t0] \n" + "mov %b[t0],0x07(%[dst]) \n" - "movzb 0x4(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x4(%3) \n" - "movzb 0x5(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x5(%3) \n" - "movzb 0x6(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x6(%3) \n" - "movzb 0x7(%2),%0 \n" - "mov %b0,0x7(%3) \n" + "vpextrw $2,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x08(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x08(%[dst]) \n" + "movzbl 0x09(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x09(%[dst]) \n" + "movzbl 0x0a(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x0a(%[dst]) \n" + "movzbl 0x0b(%[src]),%k[t0] \n" + "mov %b[t0],0x0b(%[dst]) \n" - "movd %%xmm0,%k1 \n" // 32 bit offset - "add %5,%1 \n" - "pshufd $0x39,%%xmm0,%%xmm0 \n" + "vpextrw $3,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x0c(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x0c(%[dst]) \n" + "movzbl 0x0d(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x0d(%[dst]) \n" + "movzbl 0x0e(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x0e(%[dst]) \n" + "movzbl 0x0f(%[src]),%k[t0] \n" + "mov %b[t0],0x0f(%[dst]) \n" - "movzb 0x8(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x8(%3) \n" - "movzb 0x9(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0x9(%3) \n" - "movzb 0xa(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0xa(%3) \n" - "movzb 0xb(%2),%0 \n" - "mov %b0,0xb(%3) \n" + "vpextrw $4,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x10(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x10(%[dst]) \n" + "movzbl 0x11(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x11(%[dst]) \n" + "movzbl 0x12(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x12(%[dst]) \n" + "movzbl 0x13(%[src]),%k[t0] \n" + "mov %b[t0],0x13(%[dst]) \n" - "movd %%xmm0,%k1 \n" // 32 bit offset - "add %5,%1 \n" + "vpextrw $5,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x14(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x14(%[dst]) \n" + "movzbl 0x15(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x15(%[dst]) \n" + "movzbl 0x16(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x16(%[dst]) \n" + "movzbl 0x17(%[src]),%k[t0] \n" + "mov %b[t0],0x17(%[dst]) \n" - "movzb 0xc(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0xc(%3) \n" - "movzb 0xd(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0xd(%3) \n" - "movzb 0xe(%2),%0 \n" - "movzb 0x00(%1,%0,1),%0 \n" - "mov %b0,0xe(%3) \n" - "movzb 0xf(%2),%0 \n" - "mov %b0,0xf(%3) \n" - "lea 0x10(%2),%2 \n" - "lea 0x10(%3),%3 \n" - "sub $0x4,%4 \n" - "jg 1b \n" - : "=&d"(pixel_temp), // %0 - "=&a"(table_temp), // %1 - "+r"(src_argb), // %2 - "+r"(dst_argb), // %3 - "+rm"(width) // %4 - : "r"(luma), // %5 - "rm"(lumacoeff) // %6 - : "memory", "cc", "xmm0", "xmm3", "xmm4", "xmm5"); + "vpextrw $6,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x18(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x18(%[dst]) \n" + "movzbl 0x19(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x19(%[dst]) \n" + "movzbl 0x1a(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x1a(%[dst]) \n" + "movzbl 0x1b(%[src]),%k[t0] \n" + "mov %b[t0],0x1b(%[dst]) \n" + + "vpextrw $7,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x1c(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x1c(%[dst]) \n" + "movzbl 0x1d(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x1d(%[dst]) \n" + "movzbl 0x1e(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x1e(%[dst]) \n" + "movzbl 0x1f(%[src]),%k[t0] \n" + "mov %b[t0],0x1f(%[dst]) \n" + + "lea 0x20(%[src]),%[src] \n" + "lea 0x20(%[dst]),%[dst] \n" + "subl $0x8,%[width] \n" + "jge 1b \n" + + "2: \n" + "addl $0x8,%[width] \n" + "jle 99f \n" + + LABELALIGN + "3: \n" + "vmovd (%[src]),%%xmm0 \n" + "vpmaddubsw %%xmm5,%%xmm0,%%xmm1 \n" + "vphaddw %%xmm1,%%xmm1,%%xmm1 \n" + "vpand %%xmm4,%%xmm1,%%xmm1 \n" + "vpextrw $0,%%xmm1,%k[t1] \n" + "add %[luma],%[t1] \n" + "movzbl 0x00(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x00(%[dst]) \n" + "movzbl 0x01(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x01(%[dst]) \n" + "movzbl 0x02(%[src]),%k[t0] \n" + "movzbl (%[t1],%[t0],1),%k[t0] \n" + "mov %b[t0],0x02(%[dst]) \n" + "movzbl 0x03(%[src]),%k[t0] \n" + "mov %b[t0],0x03(%[dst]) \n" + "lea 0x04(%[src]),%[src] \n" + "lea 0x04(%[dst]),%[dst] \n" + "subl $0x1,%[width] \n" + "jg 3b \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), +#if defined(__i386__) + [width] "+m"(width), +#else + [width] "+rm"(width), +#endif + [t0] "=&q"(pixel_temp), [t1] "=&r"(table_temp) + : [luma] "r"(luma), [coeff] "m"(coeff) + : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5"); } -#endif // HAS_ARGBLUMACOLORTABLEROW_SSSE3 +#endif // HAS_ARGBLUMACOLORTABLEROW_AVX2 static const uvec8 kYUV24Shuffle[3] = { {8, 9, 0, 8, 9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12}, @@ -12278,7 +14394,7 @@ void ClampFloatToZero_SSE2(const float* src_x, float* dst_y, int width) { : "memory", "cc", "xmm0", "xmm1"); } -#ifdef HAS_CONVERT16TO8ROW_AVX2 +#ifdef HAS_CONVERT8TO8ROW_AVX2 void Convert8To8Row_AVX2(const uint8_t* src_y, uint8_t* dst_y, int scale, @@ -12315,7 +14431,62 @@ void Convert8To8Row_AVX2(const uint8_t* src_y, "r"(bias) // %4 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); } -#endif // HAS_CONVERT16TO8ROW_AVX2 +#endif // HAS_CONVERT8TO8ROW_AVX2 + +#ifdef HAS_CONVERT8TO8ROW_AVX512BW +void Convert8To8Row_AVX512BW(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width) { + uintptr_t temp; + asm volatile( + "vpbroadcastw %k[scale],%%zmm2 \n" + "vpbroadcastb %k[bias],%%zmm3 \n" + "vpxord %%zmm4,%%zmm4,%%zmm4 \n" + "vpsllw $8,%%zmm2,%%zmm2 \n" + "sub $0x40,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu8 (%[src]),%%zmm0 \n" + "lea 0x40(%[src]),%[src] \n" + "vpunpckhbw %%zmm4,%%zmm0,%%zmm1 \n" + "vpunpcklbw %%zmm4,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddb %%zmm3,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst]) \n" + "lea 0x40(%[dst]),%[dst] \n" + "sub $0x40,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x40,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovq %[temp],%%k1 \n" + "vmovdqu8 (%[src]),%%zmm0%{%%k1%}%{z%} \n" + "vpunpckhbw %%zmm4,%%zmm0,%%zmm1 \n" + "vpunpcklbw %%zmm4,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm0,%%zmm0 \n" + "vpmulhuw %%zmm2,%%zmm1,%%zmm1 \n" + "vpackuswb %%zmm1,%%zmm0,%%zmm0 \n" + "vpaddb %%zmm3,%%zmm0,%%zmm0 \n" + "vmovdqu8 %%zmm0,(%[dst])%{%%k1%} \n" + + "99: \n" + "vzeroupper \n" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [width] "+r"(width), + [temp] "=&r"(temp) + : [scale] "r"(scale), [bias] "r"(bias) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "k1"); +} +#endif // HAS_CONVERT8TO8ROW_AVX512BW // =========================================================================== // Module 1: AYUV to Planar Y / UV / VU (AVX512VBMI) @@ -12491,10 +14662,155 @@ void OMITFP AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, } #endif // HAS_AYUVTOVUROW_AVX512VBMI +#ifdef HAS_AYUVTOYROW_AVX2 +void OMITFP AYUVToYRow_AVX2(const uint8_t* src_ayuv, + uint8_t* dst_y, + int width) { + asm volatile( + "vpcmpeqd %%ymm2,%%ymm2,%%ymm2 \n" + "vpsrld $24,%%ymm2,%%ymm2 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vmovdqu 0x20(%[src]),%%ymm1 \n" + "lea 0x40(%[src]),%[src] \n" + "vpsrld $16,%%ymm0,%%ymm0 \n" + "vpsrld $16,%%ymm1,%%ymm1 \n" + "vpand %%ymm2,%%ymm0,%%ymm0 \n" + "vpand %%ymm2,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vextracti128 $1,%%ymm0,%%xmm1 \n" + "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "sub $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ayuv), [dst] "+r"(dst_y), [width] "+r"(width) + : + : "memory", "cc", "xmm0", "xmm1", "xmm2"); +} +#endif // HAS_AYUVTOYROW_AVX2 + +#if defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2) +static const uint8_t kShufAYUVToUV16[16] = { + 1, 5, 0, 4, 9, 13, 8, 12, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +static const uint8_t kShufAYUVToVU16[16] = { + 0, 4, 1, 5, 8, 12, 9, 13, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +static inline void AYUVToUVOrVURow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + const uint8_t* shuf_table, + int width) { + asm volatile( + "vbroadcasti128 (%[shuf]),%%ymm6 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpsrlw $15,%%ymm7,%%ymm7 \n" + "vpsllw $8,%%ymm7,%%ymm5 \n" + "vpor %%ymm5,%%ymm7,%%ymm7 \n" + "vpxor %%ymm5,%%ymm5,%%ymm5 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "vmovdqu 0x20(%[src]),%%ymm1 \n" + "vmovdqu (%[src],%[stride]),%%ymm2 \n" + "vmovdqu 0x20(%[src],%[stride]),%%ymm3 \n" + "lea 0x40(%[src]),%[src] \n" + "vpshufb %%ymm6,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm6,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm6,%%ymm2,%%ymm2 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm3 \n" + "vpunpcklqdq %%ymm1,%%ymm0,%%ymm0 \n" + "vpunpcklqdq %%ymm3,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm7,%%ymm0,%%ymm0 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n" + "vpaddw %%ymm2,%%ymm0,%%ymm0 \n" + "vpsrlw $1,%%ymm0,%%ymm0 \n" + "vpavgw %%ymm5,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vextracti128 $1,%%ymm0,%%xmm1 \n" + "vpackuswb %%xmm1,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "subl $0x10,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ayuv), [dst] "+r"(dst_uv), [width] "+rm"(width) + : [stride] "r"((ptrdiff_t)src_stride_ayuv), [shuf] "r"(shuf_table) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm5", "xmm6", "xmm7"); +} +#endif // defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2) + +#ifdef HAS_AYUVTOUVROW_AVX2 +void OMITFP AYUVToUVRow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVOrVURow_AVX2(src_ayuv, src_stride_ayuv, dst_uv, kShufAYUVToUV16, + width); +} +#endif // HAS_AYUVTOUVROW_AVX2 + +#ifdef HAS_AYUVTOVUROW_AVX2 +void OMITFP AYUVToVURow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToUVOrVURow_AVX2(src_ayuv, src_stride_ayuv, dst_vu, kShufAYUVToVU16, + width); +} +#endif // HAS_AYUVTOVUROW_AVX2 + // =========================================================================== -// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX512BW) +// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX2 / AVX512BW) // =========================================================================== +#ifdef HAS_AR30TOARGBROW_AVX2 +void OMITFP AR30ToARGBRow_AVX2(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + static const uint32_t kC0000000 = 0xc0000000u; + static const uint32_t k300000FF = 0x300000ffu; + static const uint32_t k0C00FF00 = 0x0c00ff00u; + static const uint32_t k03FF0000 = 0x03ff0000u; + asm volatile( + "vpbroadcastd %[m0],%%ymm4 \n" + "vpbroadcastd %[m1],%%ymm5 \n" + "vpbroadcastd %[m2],%%ymm6 \n" + "vpbroadcastd %[m3],%%ymm7 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpsrld $2,%%ymm0,%%ymm1 \n" + "vpsrld $4,%%ymm0,%%ymm2 \n" + "vpsrld $6,%%ymm0,%%ymm3 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + "vpand %%ymm7,%%ymm3,%%ymm3 \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + "vpor %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_argb), [width] "+r"(width) + : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00), + [m3] "m"(k03FF0000) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_AR30TOARGBROW_AVX2 + #ifdef HAS_AR30TOARGBROW_AVX512BW void OMITFP AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_argb, @@ -12561,10 +14877,55 @@ void OMITFP AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, } #endif // HAS_AR30TOARGBROW_AVX512BW -#ifdef HAS_AR30TOABGRROW_AVX512BW +#if defined(HAS_AR30TOABGRROW_AVX2) || defined(HAS_AR30TOABGRROW_AVX512BW) static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7, 10, 9, 8, 11, 14, 13, 12, 15}; +#endif +#ifdef HAS_AR30TOABGRROW_AVX2 +void OMITFP AR30ToABGRRow_AVX2(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + static const uint32_t kC0000000 = 0xc0000000u; + static const uint32_t k300000FF = 0x300000ffu; + static const uint32_t k0C00FF00 = 0x0c00ff00u; + static const uint32_t k03FF0000 = 0x03ff0000u; + asm volatile( + "vpbroadcastd %[m0],%%ymm4 \n" + "vpbroadcastd %[m1],%%ymm5 \n" + "vpbroadcastd %[m2],%%ymm6 \n" + "vbroadcasti128 %[shuf],%%ymm7 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpsrld $2,%%ymm0,%%ymm1 \n" + "vpsrld $4,%%ymm0,%%ymm2 \n" + "vpand %%ymm5,%%ymm1,%%ymm1 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + "vpsrld $6,%%ymm0,%%ymm3 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vpbroadcastd %[m3],%%ymm1 \n" + "vpand %%ymm1,%%ymm3,%%ymm3 \n" + "vpor %%ymm3,%%ymm2,%%ymm2 \n" + "vpor %%ymm2,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm7,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_abgr), [width] "+r"(width) + : [m0] "m"(kC0000000), [m1] "m"(k300000FF), [m2] "m"(k0C00FF00), + [m3] "m"(k03FF0000), [shuf] "m"(kShufSwapRB16) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_AR30TOABGRROW_AVX2 + +#ifdef HAS_AR30TOABGRROW_AVX512BW void OMITFP AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_abgr, int width) { @@ -12633,6 +14994,40 @@ void OMITFP AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, } #endif // HAS_AR30TOABGRROW_AVX512BW +#ifdef HAS_AR30TOAB30ROW_AVX2 +void OMITFP AR30ToAB30Row_AVX2(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + static const uint32_t kC00FFC00 = 0xc00ffc00u; + static const uint32_t k3FF00000 = 0x3ff00000u; + static const uint32_t k000003FF = 0x000003ffu; + asm volatile( + "vpbroadcastd %[m0],%%ymm3 \n" + "vpbroadcastd %[m1],%%ymm4 \n" + "vpbroadcastd %[m2],%%ymm5 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src]),%%ymm0 \n" + "lea 0x20(%[src]),%[src] \n" + "vpslld $20,%%ymm0,%%ymm1 \n" + "vpsrld $20,%%ymm0,%%ymm2 \n" + "vpand %%ymm3,%%ymm0,%%ymm0 \n" + "vpand %%ymm4,%%ymm1,%%ymm1 \n" + "vpand %%ymm5,%%ymm2,%%ymm2 \n" + "vpor %%ymm1,%%ymm0,%%ymm0 \n" + "vpor %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" + "jg 1b \n" + "vzeroupper \n" + : [src] "+r"(src_ar30), [dst] "+r"(dst_ab30), [width] "+r"(width) + : [m0] "m"(kC00FFC00), [m1] "m"(k3FF00000), [m2] "m"(k000003FF) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); +} +#endif // HAS_AR30TOAB30ROW_AVX2 + #ifdef HAS_AR30TOAB30ROW_AVX512BW void OMITFP AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, uint8_t* dst_ab30, @@ -13864,12 +16259,6 @@ void OMITFP ARGBMultiplyRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBMULTIPLYROW_AVX512BW -#if defined(HAS_ARGBGRAYROW_AVX512BW) || defined(HAS_ARGBGRAYROW_AVX512VNNI) -static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, - 8, 8, 8, 0x80, 12, 12, 12, 0x80}; -#endif // defined(HAS_ARGBGRAYROW_AVX512BW) || - // defined(HAS_ARGBGRAYROW_AVX512VNNI) - #ifdef HAS_ARGBGRAYROW_AVX512BW void OMITFP ARGBGrayRow_AVX512BW(const uint8_t* src_argb, uint8_t* dst_argb, @@ -14490,6 +16879,1163 @@ void OMITFP ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb, } #endif // HAS_ARGBALPHABLENDROW_AVX512BW +// =========================================================================== +// Module 6: Extended YUV (I444, YUY2, UYVY, I210, I212, I410, P210, P410) +// to ARGB / RGBA / RGB24 / AR30 (AVX512BW / AVX512VBMI) +// =========================================================================== + +#define READYUV444_AVX512BW \ + "vpmovzxbw (%[u_buf]),%%zmm3 \n" \ + "vpmovzxbw 0x00(%[u_buf],%[v_buf],1),%%zmm1 \n" \ + "lea 0x20(%[u_buf]),%[u_buf] \n" \ + "vpsllw $8,%%zmm1,%%zmm1 \n" \ + "vpord %%zmm1,%%zmm3,%%zmm3 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4 \n" \ + "lea 0x20(%[y_buf]),%[y_buf] \n" \ + "vpsllw $8,%%zmm4,%%zmm1 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV444_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "mov $-1,%[temp] \n" \ + "vpmovzxbw (%[u_buf]),%%zmm3%{%%k1%}%{z%} \n" \ + "vpmovzxbw 0x00(%[u_buf],%[v_buf],1),%%zmm1%{%%k1%}%{z%} \n" \ + "vpsllw $8,%%zmm1,%%zmm1 \n" \ + "vpord %%zmm1,%%zmm3,%%zmm3 \n" \ + "vpmovzxbw (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "vpsllw $8,%%zmm4,%%zmm1 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define STORERGBA_AVX512BW \ + "vpunpcklbw %%zmm0,%%zmm5,%%zmm3 \n" \ + "vpermq %%zmm3,%%zmm18,%%zmm3 \n" \ + "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n" \ + "vpermq %%zmm1,%%zmm18,%%zmm1 \n" \ + "vpunpcklwd %%zmm1,%%zmm3,%%zmm0 \n" \ + "vpunpckhwd %%zmm1,%%zmm3,%%zmm1 \n" \ + "vmovups %%zmm0,(%[dst_rgba]) \n" \ + "vmovups %%zmm1,0x40(%[dst_rgba]) \n" \ + "lea 0x80(%[dst_rgba]),%[dst_rgba] \n" + +#define STORERGBA_MASK_AVX512BW \ + "vpunpcklbw %%zmm0,%%zmm5,%%zmm3 \n" \ + "vpermq %%zmm3,%%zmm18,%%zmm3 \n" \ + "vpunpcklbw %%zmm2,%%zmm1,%%zmm1 \n" \ + "vpermq %%zmm1,%%zmm18,%%zmm1 \n" \ + "vpunpcklwd %%zmm1,%%zmm3,%%zmm0 \n" \ + "vpunpckhwd %%zmm1,%%zmm3,%%zmm1 \n" \ + "kshiftrd $16,%%k1,%%k2 \n" \ + "vmovdqu32 %%zmm0,(%[dst_rgba])%{%%k1%} \n" \ + "vmovdqu32 %%zmm1,0x40(%[dst_rgba])%{%%k2%} \n" + +#define READYUV210_AVX512BW \ + "vmovdqu16 (%[u_buf]),%%ymm3 \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1 \n" \ + "lea 0x20(%[u_buf]),%[u_buf] \n" \ + "vpsrlw $2,%%ymm3,%%ymm3 \n" \ + "vpsrlw $2,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vpsllw $6,%%zmm4,%%zmm1 \n" \ + "vpsrlw $4,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV210_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu16 (%[u_buf]),%%ymm3%{%%k2%}%{z%} \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1%{%%k2%}%{z%} \n" \ + "vpsrlw $2,%%ymm3,%%ymm3 \n" \ + "vpsrlw $2,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \ + "vpsllw $6,%%zmm4,%%zmm1 \n" \ + "vpsrlw $4,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV212_AVX512BW \ + "vmovdqu16 (%[u_buf]),%%ymm3 \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1 \n" \ + "lea 0x20(%[u_buf]),%[u_buf] \n" \ + "vpsraw $4,%%ymm3,%%ymm3 \n" \ + "vpsraw $4,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vpsllw $4,%%zmm4,%%zmm1 \n" \ + "vpsrlw $8,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV212_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu16 (%[u_buf]),%%ymm3%{%%k2%}%{z%} \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%ymm1%{%%k2%}%{z%} \n" \ + "vpsraw $4,%%ymm3,%%ymm3 \n" \ + "vpsraw $4,%%ymm1,%%ymm1 \n" \ + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" \ + "vpackuswb %%ymm1,%%ymm1,%%ymm1 \n" \ + "vpunpcklbw %%ymm1,%%ymm3,%%ymm3 \n" \ + "vpermw %%zmm3,%%zmm16,%%zmm3 \n" \ + "vpsllw $4,%%zmm4,%%zmm1 \n" \ + "vpsrlw $8,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV410_AVX512BW \ + "vmovdqu16 (%[u_buf]),%%zmm3 \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%zmm1 \n" \ + "lea 0x40(%[u_buf]),%[u_buf] \n" \ + "vpsrlw $2,%%zmm3,%%zmm3 \n" \ + "vpsrlw $2,%%zmm1,%%zmm1 \n" \ + "vpackuswb %%zmm3,%%zmm3,%%zmm3 \n" \ + "vpackuswb %%zmm1,%%zmm1,%%zmm1 \n" \ + "vpunpcklbw %%zmm1,%%zmm3,%%zmm3 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vpsllw $6,%%zmm4,%%zmm1 \n" \ + "vpsrlw $4,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READYUV410_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vmovdqu16 (%[u_buf]),%%zmm3%{%%k1%}%{z%} \n" \ + "vmovdqu16 0x00(%[u_buf],%[v_buf],1),%%zmm1%{%%k1%}%{z%} \n" \ + "vpsrlw $2,%%zmm3,%%zmm3 \n" \ + "vpsrlw $2,%%zmm1,%%zmm1 \n" \ + "vpackuswb %%zmm3,%%zmm3,%%zmm3 \n" \ + "vpackuswb %%zmm1,%%zmm1,%%zmm1 \n" \ + "vpunpcklbw %%zmm1,%%zmm3,%%zmm3 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "vpsllw $6,%%zmm4,%%zmm1 \n" \ + "vpsrlw $4,%%zmm4,%%zmm4 \n" \ + "vpord %%zmm1,%%zmm4,%%zmm4 \n" + +#define READP210_AVX512BW \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vmovdqu32 (%[uv_buf]),%%zmm3 \n" \ + "lea 0x40(%[uv_buf]),%[uv_buf] \n" \ + "vpshufb %%zmm14,%%zmm3,%%zmm3 \n" + +#define READP210_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "mov $-1,%[temp] \n" \ + "lea 1(%q[width]),%k[y_buf] \n" \ + "shr $1,%k[y_buf] \n" \ + "bzhi %q[y_buf],%[temp],%[y_buf] \n" \ + "kmovw %k[y_buf],%%k2 \n" \ + "vmovdqu32 (%[uv_buf]),%%zmm3%{%%k2%}%{z%} \n" \ + "vpshufb %%zmm14,%%zmm3,%%zmm3 \n" + +#define READP410_AVX512BW \ + "vmovdqu16 (%[y_buf]),%%zmm4 \n" \ + "lea 0x40(%[y_buf]),%[y_buf] \n" \ + "vmovdqu32 (%[uv_buf]),%%zmm3 \n" \ + "vmovdqu32 0x40(%[uv_buf]),%%zmm1 \n" \ + "lea 0x80(%[uv_buf]),%[uv_buf] \n" \ + "vpsrlw $8,%%zmm3,%%zmm3 \n" \ + "vpsrlw $8,%%zmm1,%%zmm1 \n" \ + "vpmovwb %%zmm3,%%ymm3 \n" \ + "vpmovwb %%zmm1,%%ymm1 \n" \ + "vinserti64x4 $1,%%ymm1,%%zmm3,%%zmm3 \n" + +#define READP410_MASK_AVX512BW \ + "mov $-1,%[temp] \n" \ + "bzhi %q[width],%[temp],%[temp] \n" \ + "kmovd %k[temp],%%k1 \n" \ + "kshiftrd $16,%%k1,%%k2 \n" \ + "vmovdqu16 (%[y_buf]),%%zmm4%{%%k1%}%{z%} \n" \ + "vmovdqu32 (%[uv_buf]),%%zmm3%{%%k1%}%{z%} \n" \ + "vmovdqu32 0x40(%[uv_buf]),%%zmm1%{%%k2%}%{z%} \n" \ + "vpsrlw $8,%%zmm3,%%zmm3 \n" \ + "vpsrlw $8,%%zmm1,%%zmm1 \n" \ + "vpmovwb %%zmm3,%%ymm3 \n" \ + "vpmovwb %%zmm1,%%ymm1 \n" \ + "vinserti64x4 $1,%%ymm1,%%zmm3,%%zmm3 \n" + +#ifdef HAS_I444TOARGBROW_AVX512BW +void OMITFP I444ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV444_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV444_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I444TOARGBROW_AVX512BW + +#ifdef HAS_I444ALPHATOARGBROW_AVX512BW +void OMITFP I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV444_AVX512BW + "vpmovzxbw (%[a_buf]),%%zmm5 \n" + "lea 0x20(%[a_buf]),%[a_buf] \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV444_MASK_AVX512BW + "vpmovzxbw (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [a_buf] "+r"(a_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I444ALPHATOARGBROW_AVX512BW + +#ifdef HAS_I444TORGB24ROW_AVX512VBMI +void OMITFP I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vmovdqu32 %[kMaskBG],%%zmm20 \n" + "vmovdqu32 %[kMaskDST0],%%zmm21 \n" + "vmovdqu32 %[kMaskDST1],%%zmm22 \n" + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV444_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_AVX512VBMI + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV444_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGB24_MASK_AVX512VBMI + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_rgb24] "+r"(dst_rgb24), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kMaskBG] "m"(kMaskBG), + [kMaskDST0] "m"(kMaskDST0), + [kMaskDST1] "m"(kMaskDST1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm20", "xmm21", "xmm22", "k1", "k3", "k4"); +} +#endif // HAS_I444TORGB24ROW_AVX512VBMI + +#ifdef HAS_I422TORGBAROW_AVX512BW +void OMITFP I422ToRGBARow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV422_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGBA_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV422_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STORERGBA_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_rgba] "+r"(dst_rgba), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I422TORGBAROW_AVX512BW + +#ifdef HAS_YUY2TOARGBROW_AVX512BW +void OMITFP YUY2ToARGBRow_AVX512BW(const uint8_t* src_yuy2, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShufY],%%zmm14 \n" + "vbroadcasti32x4 %[kShufUV],%%zmm15 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src_yuy2]),%%zmm1 \n" + "lea 0x40(%[src_yuy2]),%[src_yuy2] \n" + "vpshufb %%zmm14,%%zmm1,%%zmm4 \n" + "vpshufb %%zmm15,%%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "mov $-1,%[temp] \n" + "add $1,%k[width] \n" + "shr $1,%k[width] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k2 \n" + "vmovdqu32 (%[src_yuy2]),%%zmm1%{%%k2%}%{z%}\n" + "vpshufb %%zmm14,%%zmm1,%%zmm4 \n" + "vpshufb %%zmm15,%%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [src_yuy2] "+r"(src_yuy2), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kShufY] "m"(kShuffleYUY2Y), + [kShufUV] "m"(kShuffleYUY2UV) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm14", "xmm15", "k1", "k2"); +} +#endif // HAS_YUY2TOARGBROW_AVX512BW + +#ifdef HAS_UYVYTOARGBROW_AVX512BW +void OMITFP UYVYToARGBRow_AVX512BW(const uint8_t* src_uyvy, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShufY],%%zmm14 \n" + "vbroadcasti32x4 %[kShufUV],%%zmm15 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu32 (%[src_uyvy]),%%zmm1 \n" + "lea 0x40(%[src_uyvy]),%[src_uyvy] \n" + "vpshufb %%zmm14,%%zmm1,%%zmm4 \n" + "vpshufb %%zmm15,%%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + "mov $-1,%[temp] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovd %k[temp],%%k1 \n" + "mov $-1,%[temp] \n" + "add $1,%k[width] \n" + "shr $1,%k[width] \n" + "bzhi %q[width],%[temp],%[temp] \n" + "kmovw %k[temp],%%k2 \n" + "vmovdqu32 (%[src_uyvy]),%%zmm1%{%%k2%}%{z%}\n" + "vpshufb %%zmm14,%%zmm1,%%zmm4 \n" + "vpshufb %%zmm15,%%zmm1,%%zmm3 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [src_uyvy] "+r"(src_uyvy), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kShufY] "m"(kShuffleUYVYY), + [kShufUV] "m"(kShuffleUYVYUV) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm14", "xmm15", "k1", "k2"); +} +#endif // HAS_UYVYTOARGBROW_AVX512BW + +#ifdef HAS_I210TOARGBROW_AVX512BW +void OMITFP I210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV210_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV210_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I210TOARGBROW_AVX512BW + +#ifdef HAS_I210TOAR30ROW_AVX512BW +void OMITFP I210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV210_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV210_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2"); +} +#endif // HAS_I210TOAR30ROW_AVX512BW + +#ifdef HAS_I210ALPHATOARGBROW_AVX512BW +void OMITFP I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV210_AVX512BW + "vmovdqu16 (%[a_buf]),%%zmm5 \n" + "lea 0x40(%[a_buf]),%[a_buf] \n" + "vpsrlw $2,%%zmm5,%%zmm5 \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV210_MASK_AVX512BW + "vmovdqu16 (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" + "vpsrlw $2,%%zmm5,%%zmm5 \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [a_buf] "+r"(a_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I210ALPHATOARGBROW_AVX512BW + +#ifdef HAS_I212TOARGBROW_AVX512BW +void OMITFP I212ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV212_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV212_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I212TOARGBROW_AVX512BW + +#ifdef HAS_I212TOAR30ROW_AVX512BW +void OMITFP I212ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV212_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV212_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2"); +} +#endif // HAS_I212TOAR30ROW_AVX512BW + +#ifdef HAS_I410TOARGBROW_AVX512BW +void OMITFP I410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV410_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV410_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I410TOARGBROW_AVX512BW + +#ifdef HAS_I410TOAR30ROW_AVX512BW +void OMITFP I410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV410_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV410_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2"); +} +#endif // HAS_I410TOAR30ROW_AVX512BW + +#ifdef HAS_I410ALPHATOARGBROW_AVX512BW +void OMITFP I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "sub %[u_buf],%[v_buf] \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READYUV410_AVX512BW + "vmovdqu16 (%[a_buf]),%%zmm5 \n" + "lea 0x40(%[a_buf]),%[a_buf] \n" + "vpsrlw $2,%%zmm5,%%zmm5 \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READYUV410_MASK_AVX512BW + "vmovdqu16 (%[a_buf]),%%zmm5%{%%k1%}%{z%}\n" + "vpsrlw $2,%%zmm5,%%zmm5 \n" + "vpackuswb %%zmm5,%%zmm5,%%zmm5 \n" + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [u_buf] "+r"(u_buf), + [v_buf] "+r"(v_buf), + [a_buf] "+r"(a_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_I410ALPHATOARGBROW_AVX512BW + +#ifdef HAS_P210TOARGBROW_AVX512BW +void OMITFP P210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShufUV],%%zmm14 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READP210_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READP210_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [kShufUV] "m"(kShuffleYUY2UV) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm14", "k1", "k2"); +} +#endif // HAS_P210TOARGBROW_AVX512BW + +#ifdef HAS_P210TOAR30ROW_AVX512BW +void OMITFP P210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vbroadcasti32x4 %[kShufUV],%%zmm14 \n" + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READP210_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READP210_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1), + [kShufUV] "m"(kShuffleYUY2UV) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm14", "xmm19", "xmm20", "xmm21", + "k1", "k2"); +} +#endif // HAS_P210TOAR30ROW_AVX512BW + +#ifdef HAS_P410TOARGBROW_AVX512BW +void OMITFP P410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READP410_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READP410_MASK_AVX512BW + YUVTORGB_AVX512BW(yuvconstants) + STOREARGB_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_argb] "+r"(dst_argb), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "k1", "k2"); +} +#endif // HAS_P410TOARGBROW_AVX512BW + +#ifdef HAS_P410TOAR30ROW_AVX512BW +void OMITFP P410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + uintptr_t temp; + asm volatile( + YUVTORGB_SETUP_AVX512BW(yuvconstants) + "vpternlogd $0xff,%%zmm5,%%zmm5,%%zmm5 \n" + "vpsrlw $6,%%zmm5,%%zmm7 \n" + "vpsllw $4,%%zmm7,%%zmm7 \n" + "vpsllw $10,%%zmm5,%%zmm19 \n" + "vpsllw $14,%%zmm5,%%zmm5 \n" + "vpxord %%zmm6,%%zmm6,%%zmm6 \n" + "vmovdqu16 (%[permar30_0]),%%zmm20 \n" + "vmovdqu16 (%[permar30_1]),%%zmm21 \n" + "sub $0x20,%[width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + READP410_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_AVX512BW + "sub $0x20,%[width] \n" + "jge 1b \n" + + "2: \n" + "add $0x20,%[width] \n" + "je 99f \n" + + READP410_MASK_AVX512BW + YUVTORGB16_AVX512BW(yuvconstants) + STOREAR30_MASK_AVX512BW + + "99: \n" + "vzeroupper \n" + : [y_buf] "+r"(y_buf), + [uv_buf] "+r"(uv_buf), + [dst_ar30] "+r"(dst_ar30), + [width] "+r"(width), + [temp] "=&r"(temp) + : [yuvconstants] "r"(yuvconstants), + [quadsplitperm] "r"(kSplitQuadWords), + [dquadsplitperm] "r"(kSplitDoubleQuadWords), + [unperm] "r"(kUnpermuteAVX512), + [permar30_0] "r"(kPermAR30_0), + [permar30_1] "r"(kPermAR30_1) + : "memory", "cc", YUVTORGB_REGS_AVX512BW "xmm0", "xmm1", "xmm2", "xmm3", + "xmm4", "xmm5", "xmm6", "xmm7", "xmm19", "xmm20", "xmm21", "k1", "k2"); +} +#endif // HAS_P410TOAR30ROW_AVX512BW + #endif // !defined(LIBYUV_DISABLE_X86) && // (defined(__x86_64__) || defined(__i386__)) && // !defined(LIBYUV_ENABLE_ROWWIN) diff --git a/source/row_win.cc b/source/row_win.cc index da55bd12e..a986d79ef 100644 --- a/source/row_win.cc +++ b/source/row_win.cc @@ -20,6 +20,7 @@ #include #include // For AVX2 intrinsics +#include // For memcpy #include // For _mm_maddubs_epi16 #if defined(_MSC_VER) && !defined(__clang__) #include @@ -32,6 +33,7 @@ extern "C" { #if defined(__clang__) || defined(__GNUC__) #define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3"))) +#define LIBYUV_TARGET_AVX __attribute__((target("avx"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) @@ -43,6 +45,7 @@ extern "C" { __attribute__((target("avx512vnni,avx512vbmi,avx512bw,avx512vl,avx512f"))) #else #define LIBYUV_TARGET_SSSE3 +#define LIBYUV_TARGET_AVX #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #define LIBYUV_TARGET_AVX512VBMI @@ -377,14 +380,16 @@ static const uint32_t kPermd1_AVX2[8] = {2, 4, 5, 6, 3, 7, 0, 1}; static const uint32_t kPermd2_AVX2[8] = {5, 6, 3, 7, 0, 1, 2, 4}; #endif // defined(HAS_RAWTORGB24ROW_AVX2) -#if defined(HAS_RAWTORGB24ROW_AVX512BW) +#if defined(HAS_RAWTORGB24ROW_AVX512BW) || \ + defined(HAS_ARGBTORGB24ROW_AVX512BW) || \ + defined(HAS_ARGBTORAWROW_AVX512BW) static const uint32_t kPermdRGB24_0_AVX512BW[16] = { 0, 1, 2, 4, 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20}; static const uint32_t kPermdRGB24_1_AVX512BW[16] = { 5, 6, 8, 9, 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25}; static const uint32_t kPermdRGB24_2_AVX512BW[16] = { 10, 12, 13, 14, 16, 17, 18, 20, 21, 22, 24, 25, 26, 28, 29, 30}; -#endif // defined(HAS_RAWTORGB24ROW_AVX512BW) +#endif #if defined(HAS_RAWTORGB24ROW_AVX512VBMI) static const uint8_t kPermRAWToRGB24_0[64] = { 0x02, 0x01, 0x00, 0x05, 0x04, 0x03, 0x08, 0x07, 0x06, 0x0b, 0x0a, @@ -689,6 +694,9 @@ void RAWToRGB24Row_AVX512VBMI(const uint8_t* src_raw, #endif // HAS_RAWTORGB24ROW_AVX512VBMI #if defined(HAS_RGBTOYMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOYMATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOUV444MATRIXROW_AVX512BW) || \ + defined(HAS_ARGBTOUVMATRIXROW_AVX512BW) || \ defined(HAS_ARGBTOYMATRIXROW_AVX512VNNI) || \ defined(HAS_ARGBTOUV444MATRIXROW_AVX512VNNI) || \ defined(HAS_ARGBTOUVMATRIXROW_AVX512VNNI) @@ -1458,21 +1466,101 @@ void MirrorUVRow_AVX2(const uint8_t* src_uv, uint8_t* dst_uv, int width) { } #endif // HAS_MIRRORUVROW_AVX2 +#ifdef HAS_MIRRORROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void MirrorRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + const __m512i zmm_shuf = _mm512_broadcast_i32x4( + _mm_setr_epi8(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); + const __m512i zmm_rev = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + const uint8_t* s = src + width; + while (width >= 64) { + s -= 64; + __m512i v = _mm512_loadu_si512((const __m512i*)s); + v = _mm512_shuffle_epi8(v, zmm_shuf); + v = _mm512_shuffle_i64x2(v, v, 0x1b); + _mm512_storeu_si512((__m512i*)dst, v); + dst += 64; + width -= 64; + } + if (width > 0) { + __mmask64 k = (1ULL << width) - 1ULL; + __m512i v = _mm512_maskz_loadu_epi8(k, src); + v = _mm512_or_si512(_mm512_srli_epi16(v, 8), _mm512_slli_epi16(v, 8)); + __m512i idx = + _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)((64 - width) >> 1))); + v = _mm512_permutexvar_epi16(idx, v); + if (width & 1) { + __m512i v_hi = _mm512_alignr_epi64(v, v, 2); + v = _mm512_alignr_epi8(v_hi, v, 1); + } + _mm512_mask_storeu_epi8(dst, k, v); + } + _mm256_zeroupper(); +} +#endif // HAS_MIRRORROW_AVX512BW + +#ifdef HAS_MIRRORSPLITUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void MirrorSplitUVRow_AVX512BW(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m512i zmm_shuf = _mm512_broadcast_i32x4( + _mm_setr_epi8(14, 12, 10, 8, 6, 4, 2, 0, 15, 13, 11, 9, 7, 5, 3, 1)); + const __m512i zmm_perm = _mm512_setr_epi64(6, 4, 2, 0, 7, 5, 3, 1); + const __m512i zmm_rev = _mm512_set_epi16( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); + const uint8_t* s = src_uv + (ptrdiff_t)width * 2; + while (width >= 32) { + s -= 64; + __m512i v = _mm512_loadu_si512((const __m512i*)s); + v = _mm512_shuffle_epi8(v, zmm_shuf); + v = _mm512_permutexvar_epi64(zmm_perm, v); + _mm256_storeu_si256((__m256i*)dst_u, _mm512_castsi512_si256(v)); + _mm256_storeu_si256((__m256i*)dst_v, _mm512_extracti64x4_epi64(v, 1)); + dst_u += 32; + dst_v += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i idx = + _mm512_sub_epi16(zmm_rev, _mm512_set1_epi16((short)(32 - width))); + __m512i v = _mm512_maskz_loadu_epi16(k, src_uv); + v = _mm512_permutexvar_epi16(idx, v); + _mm256_mask_storeu_epi8(dst_u, k, _mm512_cvtepi16_epi8(v)); + _mm256_mask_storeu_epi8(dst_v, k, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8))); + } + _mm256_zeroupper(); +} +#endif // HAS_MIRRORSPLITUVROW_AVX512BW + #ifdef HAS_MIRRORUVROW_AVX512BW LIBYUV_TARGET_AVX512BW void MirrorUVRow_AVX512BW(const uint8_t* src_uv, uint8_t* dst_uv, int width) { const __m512i zmm_idx = _mm512_set_epi16( 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31); - src_uv += width * 2; - while (width > 0) { - src_uv -= 64; - __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_uv); + const uint8_t* s = src_uv + (ptrdiff_t)width * 2; + while (width >= 32) { + s -= 64; + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)s); zmm0 = _mm512_permutexvar_epi16(zmm_idx, zmm0); _mm512_storeu_si512((__m512i*)dst_uv, zmm0); dst_uv += 64; width -= 32; } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i idx = + _mm512_sub_epi16(zmm_idx, _mm512_set1_epi16((short)(32 - width))); + __m512i zmm0 = _mm512_maskz_loadu_epi16(k, src_uv); + _mm512_mask_storeu_epi16(dst_uv, k, _mm512_permutexvar_epi16(idx, zmm0)); + } _mm256_zeroupper(); } #endif // HAS_MIRRORUVROW_AVX512BW @@ -2500,13 +2588,18 @@ void NV21ToARGBRow_AVX2(const uint8_t* src_y, } #endif // HAS_NV21TOARGBROW_AVX2 -#if defined(HAS_I422TORGB24ROW_AVX2) || defined(HAS_I422TORGB24ROW_AVX512BW) +#if defined(HAS_I422TORGB24ROW_AVX2) || \ + defined(HAS_I422TORGB24ROW_AVX512BW) || \ + defined(HAS_ARGBTORGB24ROW_AVX2) || \ + defined(HAS_ARGBTORGB24ROW_AVX512BW) static const uint8_t kShuffleMaskARGBToRGB24[2][16] = { {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 10u, 12u, 13u, 14u, 128u, 128u, 128u, 128u}, {0u, 1u, 2u, 4u, 5u, 6u, 8u, 9u, 128u, 128u, 128u, 128u, 10u, 12u, 13u, 14u}}; +#endif +#if defined(HAS_I422TORGB24ROW_AVX2) LIBYUV_TARGET_AVX2 void I422ToRGB24Row_AVX2(const uint8_t* src_y, const uint8_t* src_u, @@ -3016,53 +3109,56 @@ void I422ToAR30Row_AVX512BW(const uint8_t* src_y, } #endif // HAS_I422TOAR30ROW_AVX512BW -#ifdef HAS_ARGBBLENDROW_SSSE3 +#ifdef HAS_ARGBBLENDROW_AVX2 static const uint8_t kShuffleAlpha[16] = {3u, 0x80, 3u, 0x80, 7u, 0x80, 7u, 0x80, 11u, 0x80, 11u, 0x80, 15u, 0x80, 15u, 0x80}; -void ARGBBlendRow_SSSE3(const uint8_t* src_argb, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - __m128i shuff = _mm_loadu_si128((const __m128i*)kShuffleAlpha); - __m128i xmm_1 = _mm_set1_epi16(1); - __m128i mask_lo = _mm_set1_epi16(0x00ff); - __m128i mask_hi = _mm_set1_epi16((short)0xff00); - __m128i mask_alpha = _mm_set1_epi32((int)0xff000000); +LIBYUV_TARGET_AVX2 +void ARGBBlendRow_AVX2(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + __m256i shuff = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kShuffleAlpha)); + __m256i ymm_1 = _mm256_set1_epi16(1); + __m256i mask_lo = _mm256_set1_epi16(0x00ff); + __m256i mask_hi = _mm256_set1_epi16((short)0xff00); + __m256i mask_alpha = _mm256_set1_epi32((int)0xff000000); - while (width >= 4) { - __m128i xmm_s0 = _mm_loadu_si128((const __m128i*)src_argb); - __m128i xmm_s1 = _mm_loadu_si128((const __m128i*)src_argb1); - src_argb += 16; - src_argb1 += 16; + while (width >= 8) { + __m256i ymm_s0 = _mm256_loadu_si256((const __m256i*)src_argb0); + __m256i ymm_s1 = _mm256_loadu_si256((const __m256i*)src_argb1); + src_argb0 += 32; + src_argb1 += 32; - __m128i xmm_a = _mm_xor_si128(xmm_s0, mask_alpha); - xmm_a = _mm_shuffle_epi8(xmm_a, shuff); - xmm_a = _mm_add_epi16(xmm_a, xmm_1); + __m256i ymm_a = _mm256_xor_si256(ymm_s0, mask_alpha); + ymm_a = _mm256_shuffle_epi8(ymm_a, shuff); + ymm_a = _mm256_add_epi16(ymm_a, ymm_1); - __m128i xmm_s1_lo = _mm_and_si128(xmm_s1, mask_lo); - __m128i xmm_s1_hi = _mm_srli_epi16(xmm_s1, 8); + __m256i ymm_s1_lo = _mm256_and_si256(ymm_s1, mask_lo); + __m256i ymm_s1_hi = _mm256_srli_epi16(ymm_s1, 8); - xmm_s1_lo = _mm_mullo_epi16(xmm_s1_lo, xmm_a); - xmm_s1_hi = _mm_mullo_epi16(xmm_s1_hi, xmm_a); + ymm_s1_lo = _mm256_mullo_epi16(ymm_s1_lo, ymm_a); + ymm_s1_hi = _mm256_mullo_epi16(ymm_s1_hi, ymm_a); - xmm_s1_lo = _mm_srli_epi16(xmm_s1_lo, 8); - xmm_s1_hi = _mm_and_si128(xmm_s1_hi, mask_hi); + ymm_s1_lo = _mm256_srli_epi16(ymm_s1_lo, 8); + ymm_s1_hi = _mm256_and_si256(ymm_s1_hi, mask_hi); - __m128i res = _mm_or_si128(xmm_s0, mask_alpha); - res = _mm_adds_epu8(res, xmm_s1_lo); - res = _mm_adds_epu8(res, xmm_s1_hi); + __m256i res = _mm256_or_si256(ymm_s0, mask_alpha); + res = _mm256_adds_epu8(res, ymm_s1_lo); + res = _mm256_adds_epu8(res, ymm_s1_hi); - _mm_storeu_si128((__m128i*)dst_argb, res); - dst_argb += 16; - width -= 4; + _mm256_storeu_si256((__m256i*)dst_argb, res); + dst_argb += 32; + width -= 8; } + _mm256_zeroupper(); while (width > 0) { - uint32_t s0 = *(const uint32_t*)src_argb; + uint32_t s0 = *(const uint32_t*)src_argb0; uint32_t s1 = *(const uint32_t*)src_argb1; - src_argb += 4; + src_argb0 += 4; src_argb1 += 4; uint32_t a = 256 - (s0 >> 24); @@ -3079,7 +3175,7 @@ void ARGBBlendRow_SSSE3(const uint8_t* src_argb, --width; } } -#endif // HAS_ARGBBLENDROW_SSSE3 +#endif // HAS_ARGBBLENDROW_AVX2 #ifdef HAS_BLENDPLANEROW_SSSE3 void BlendPlaneRow_SSSE3(const uint8_t* src0, @@ -3423,9 +3519,101 @@ void RGB24MirrorRow_AVX512VBMI(const uint8_t* src_rgb24, #endif // HAS_RGB24MIRRORROW_AVX512VBMI // =========================================================================== -// Module 1: AYUV to Planar Y / UV / VU (AVX512VBMI) +// Module 1: AYUV to Planar Y / UV / VU (AVX2 / AVX512VBMI) // =========================================================================== +#ifdef HAS_AYUVTOYROW_AVX2 +LIBYUV_TARGET_AVX2 +void AYUVToYRow_AVX2(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { + const __m256i ymm_mask = _mm256_set1_epi32(0x000000ff); + while (width >= 16) { + __m256i ymm0 = _mm256_and_si256( + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_ayuv), 16), + ymm_mask); + __m256i ymm1 = _mm256_and_si256( + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)(src_ayuv + 32)), + 16), + ymm_mask); + src_ayuv += 64; + ymm0 = _mm256_permute4x64_epi64(_mm256_packus_epi32(ymm0, ymm1), 0xd8); + __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0), + _mm256_extracti128_si256(ymm0, 1)); + _mm_storeu_si128((__m128i*)dst_y, out); + dst_y += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_AYUVTOYROW_AVX2 + +#if defined(HAS_AYUVTOUVROW_AVX2) || defined(HAS_AYUVTOVUROW_AVX2) +static const uint8_t kShufAYUVToUV16_Win[16] = { + 1, 5, 0, 4, 9, 13, 8, 12, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +static const uint8_t kShufAYUVToVU16_Win[16] = { + 0, 4, 1, 5, 8, 12, 9, 13, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; + +LIBYUV_TARGET_AVX2 +static inline void AYUVToUVOrVURow_Win_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + const uint8_t* shuf_table, + int width) { + const __m256i ymm_shuf = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuf_table)); + const __m256i ymm_0101 = _mm256_set1_epi16(0x0101); + const __m256i ymm_zero = _mm256_setzero_si256(); + while (width >= 16) { + __m256i s00 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)src_ayuv), ymm_shuf); + __m256i s01 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ayuv + 32)), ymm_shuf); + __m256i s10 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ayuv + src_stride_ayuv)), + ymm_shuf); + __m256i s11 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ayuv + src_stride_ayuv + 32)), + ymm_shuf); + src_ayuv += 64; + __m256i p0 = + _mm256_maddubs_epi16(_mm256_unpacklo_epi64(s00, s01), ymm_0101); + __m256i p1 = + _mm256_maddubs_epi16(_mm256_unpacklo_epi64(s10, s11), ymm_0101); + p0 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(p0, p1), 1), + ymm_zero); + p0 = _mm256_permute4x64_epi64(p0, 0xd8); + __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(p0), + _mm256_extracti128_si256(p0, 1)); + _mm_storeu_si128((__m128i*)dst_uv, out); + dst_uv += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif + +#ifdef HAS_AYUVTOUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void AYUVToUVRow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_uv, + int width) { + AYUVToUVOrVURow_Win_AVX2(src_ayuv, src_stride_ayuv, dst_uv, + kShufAYUVToUV16_Win, width); +} +#endif // HAS_AYUVTOUVROW_AVX2 + +#ifdef HAS_AYUVTOVUROW_AVX2 +LIBYUV_TARGET_AVX2 +void AYUVToVURow_AVX2(const uint8_t* src_ayuv, + int src_stride_ayuv, + uint8_t* dst_vu, + int width) { + AYUVToUVOrVURow_Win_AVX2(src_ayuv, src_stride_ayuv, dst_vu, + kShufAYUVToVU16_Win, width); +} +#endif // HAS_AYUVTOVUROW_AVX2 + #ifdef HAS_AYUVTOYROW_AVX512VBMI LIBYUV_TARGET_AVX512VBMI void AYUVToYRow_AVX512VBMI(const uint8_t* src_ayuv, uint8_t* dst_y, int width) { @@ -3548,9 +3736,34 @@ void AYUVToVURow_AVX512VBMI(const uint8_t* src_ayuv, #endif // HAS_AYUVTOVUROW_AVX512VBMI // =========================================================================== -// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX512BW) +// Module 2: 10-bit / 30-bit AR30 & XR30 Pack / Unpack (AVX2 / AVX512BW) // =========================================================================== +#ifdef HAS_AR30TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void AR30ToARGBRow_AVX2(const uint8_t* src_ar30, + uint8_t* dst_argb, + int width) { + const __m256i m0 = _mm256_set1_epi32((int)0xc0000000u); + const __m256i m1 = _mm256_set1_epi32(0x300000ff); + const __m256i m2 = _mm256_set1_epi32(0x0c00ff00); + const __m256i m3 = _mm256_set1_epi32(0x03ff0000); + while (width >= 8) { + __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30); + src_ar30 += 32; + __m256i out = _mm256_or_si256( + _mm256_or_si256(_mm256_and_si256(v, m0), + _mm256_and_si256(_mm256_srli_epi32(v, 2), m1)), + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v, 4), m2), + _mm256_and_si256(_mm256_srli_epi32(v, 6), m3))); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOARGBROW_AVX2 + #ifdef HAS_AR30TOARGBROW_AVX512BW LIBYUV_TARGET_AVX512BW void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, @@ -3586,6 +3799,36 @@ void AR30ToARGBRow_AVX512BW(const uint8_t* src_ar30, } #endif // HAS_AR30TOARGBROW_AVX512BW +#ifdef HAS_AR30TOABGRROW_AVX2 +LIBYUV_TARGET_AVX2 +void AR30ToABGRRow_AVX2(const uint8_t* src_ar30, + uint8_t* dst_abgr, + int width) { + static const uint8_t kShufSwapRB16[16] = {2, 1, 0, 3, 6, 5, 4, 7, + 10, 9, 8, 11, 14, 13, 12, 15}; + const __m256i m0 = _mm256_set1_epi32((int)0xc0000000u); + const __m256i m1 = _mm256_set1_epi32(0x300000ff); + const __m256i m2 = _mm256_set1_epi32(0x0c00ff00); + const __m256i m3 = _mm256_set1_epi32(0x03ff0000); + const __m256i shuf = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kShufSwapRB16)); + while (width >= 8) { + __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30); + src_ar30 += 32; + __m256i out = _mm256_or_si256( + _mm256_or_si256(_mm256_and_si256(v, m0), + _mm256_and_si256(_mm256_srli_epi32(v, 2), m1)), + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v, 4), m2), + _mm256_and_si256(_mm256_srli_epi32(v, 6), m3))); + out = _mm256_shuffle_epi8(out, shuf); + _mm256_storeu_si256((__m256i*)dst_abgr, out); + dst_abgr += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOABGRROW_AVX2 + #ifdef HAS_AR30TOABGRROW_AVX512BW LIBYUV_TARGET_AVX512BW void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, @@ -3627,6 +3870,29 @@ void AR30ToABGRRow_AVX512BW(const uint8_t* src_ar30, } #endif // HAS_AR30TOABGRROW_AVX512BW +#ifdef HAS_AR30TOAB30ROW_AVX2 +LIBYUV_TARGET_AVX2 +void AR30ToAB30Row_AVX2(const uint8_t* src_ar30, + uint8_t* dst_ab30, + int width) { + const __m256i m0 = _mm256_set1_epi32((int)0xc00ffc00u); + const __m256i m1 = _mm256_set1_epi32(0x3ff00000); + const __m256i m2 = _mm256_set1_epi32(0x000003ff); + while (width >= 8) { + __m256i v = _mm256_loadu_si256((const __m256i*)src_ar30); + src_ar30 += 32; + __m256i out = _mm256_or_si256( + _mm256_and_si256(v, m0), + _mm256_or_si256(_mm256_and_si256(_mm256_slli_epi32(v, 20), m1), + _mm256_and_si256(_mm256_srli_epi32(v, 20), m2))); + _mm256_storeu_si256((__m256i*)dst_ab30, out); + dst_ab30 += 32; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_AR30TOAB30ROW_AVX2 + #ifdef HAS_AR30TOAB30ROW_AVX512BW LIBYUV_TARGET_AVX512BW void AR30ToAB30Row_AVX512BW(const uint8_t* src_ar30, @@ -5177,6 +5443,4653 @@ void ARGBAlphaBlendRow_AVX512BW(const uint8_t* src_argb0, } #endif // HAS_ARGBALPHABLENDROW_AVX512BW +// =========================================================================== +// Module 6: AVX2 Row Functions (Minimum AVX2 Coverage) +// =========================================================================== + +#ifdef HAS_ARGBSEPIAROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBSepiaRow_AVX2(uint8_t* dst_argb, int width) { + static const int8_t kSepiaB[16] = {17, 68, 35, 0, 17, 68, 35, 0, + 17, 68, 35, 0, 17, 68, 35, 0}; + static const int8_t kSepiaG[16] = {22, 88, 45, 0, 22, 88, 45, 0, + 22, 88, 45, 0, 22, 88, 45, 0}; + static const int8_t kSepiaR[16] = {24, 98, 50, 0, 24, 98, 50, 0, + 24, 98, 50, 0, 24, 98, 50, 0}; + if (width >= 8) { + const __m256i vb = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaB)); + const __m256i vg = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaG)); + const __m256i vr = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)kSepiaR)); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb); + __m256i b = _mm256_maddubs_epi16(v0, vb); + __m256i g = _mm256_maddubs_epi16(v0, vg); + __m256i r = _mm256_maddubs_epi16(v0, vr); + __m256i bg = _mm256_srli_epi16(_mm256_hadd_epi16(b, g), 7); + __m256i rr = _mm256_srli_epi16(_mm256_hadd_epi16(r, r), 7); + bg = _mm256_packus_epi16(bg, bg); + rr = _mm256_packus_epi16(rr, rr); + bg = _mm256_unpacklo_epi8(bg, _mm256_shuffle_epi32(bg, 0x01)); + __m256i a = _mm256_srli_epi32(v0, 24); + a = _mm256_packus_epi16(a, a); + a = _mm256_packus_epi16(a, a); + __m256i ra = _mm256_unpacklo_epi8(rr, a); + __m256i out = _mm256_unpacklo_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBSepiaRow_C(dst_argb, width); + } +} +#endif // HAS_ARGBSEPIAROW_AVX2 + +#ifdef HAS_ARGBCOLORMATRIXROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBColorMatrixRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + static const int8_t kShufTranspose4x4[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; + if (width >= 8) { + const __m256i m = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)matrix_argb)); + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufTranspose4x4)); + const __m256i m0 = _mm256_shuffle_epi32(m, 0x00); + const __m256i m1 = _mm256_shuffle_epi32(m, 0x55); + const __m256i m2 = _mm256_shuffle_epi32(m, 0xaa); + const __m256i m3 = _mm256_shuffle_epi32(m, 0xff); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + __m256i p01 = _mm256_hadds_epi16(_mm256_maddubs_epi16(v0, m0), + _mm256_maddubs_epi16(v0, m1)); + __m256i p23 = _mm256_hadds_epi16(_mm256_maddubs_epi16(v0, m2), + _mm256_maddubs_epi16(v0, m3)); + p01 = _mm256_srai_epi16(p01, 6); + p23 = _mm256_srai_epi16(p23, 6); + __m256i out = _mm256_shuffle_epi8(_mm256_packus_epi16(p01, p23), shuf); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBColorMatrixRow_C(src_argb, dst_argb, matrix_argb, width); + } +} +#endif // HAS_ARGBCOLORMATRIXROW_AVX2 + +#ifdef HAS_ARGBQUANTIZEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBQuantizeRow_AVX2(uint8_t* dst_argb, + int scale, + int interval_size, + int interval_offset, + int width) { + if (width >= 8) { + const __m256i v_scale = _mm256_set1_epi16((short)scale); + const __m256i v_size = _mm256_set1_epi16((short)interval_size); + const __m256i v_off = _mm256_set1_epi16((short)interval_offset); + const __m256i zero = _mm256_setzero_si256(); + const __m256i alpha_mask = _mm256_set1_epi32((int)0xff000000u); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb); + __m256i a = _mm256_and_si256(v0, alpha_mask); + __m256i lo = _mm256_unpacklo_epi8(v0, zero); + __m256i hi = _mm256_unpackhi_epi8(v0, zero); + lo = _mm256_add_epi16( + _mm256_mullo_epi16(_mm256_mulhi_epu16(lo, v_scale), v_size), v_off); + hi = _mm256_add_epi16( + _mm256_mullo_epi16(_mm256_mulhi_epu16(hi, v_scale), v_size), v_off); + __m256i out = _mm256_or_si256( + _mm256_andnot_si256(alpha_mask, _mm256_packus_epi16(lo, hi)), a); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBQuantizeRow_C(dst_argb, scale, interval_size, interval_offset, width); + } +} +#endif // HAS_ARGBQUANTIZEROW_AVX2 + +#ifdef HAS_ARGBSHADEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBShadeRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + uint32_t value) { + if (width >= 8) { + __m256i v_val = _mm256_set1_epi32((int)value); + v_val = _mm256_unpacklo_epi8(v_val, v_val); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + __m256i lo = _mm256_unpacklo_epi8(v0, v0); + __m256i hi = _mm256_unpackhi_epi8(v0, v0); + lo = _mm256_srli_epi16(_mm256_mulhi_epu16(lo, v_val), 8); + hi = _mm256_srli_epi16(_mm256_mulhi_epu16(hi, v_val), 8); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_packus_epi16(lo, hi)); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBShadeRow_C(src_argb, dst_argb, width, value); + } +} +#endif // HAS_ARGBSHADEROW_AVX2 + +#ifdef HAS_SOBELXROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelXRow_AVX2(const uint8_t* src_y0, + const uint8_t* src_y1, + const uint8_t* src_y2, + uint8_t* dst_sobelx, + int width) { + while (width > 0) { + __m256i a0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y0)); + __m256i b0 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 2))); + __m256i a1 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y1)); + __m256i b1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 2))); + __m256i a2 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y2)); + __m256i b2 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y2 + 2))); + __m256i d0 = _mm256_sub_epi16(a0, b0); + __m256i d1 = _mm256_sub_epi16(a1, b1); + __m256i d2 = _mm256_sub_epi16(a2, b2); + __m256i sum = + _mm256_add_epi16(_mm256_add_epi16(d0, d2), _mm256_add_epi16(d1, d1)); + sum = _mm256_abs_epi16(sum); + sum = _mm256_permute4x64_epi64(_mm256_packus_epi16(sum, sum), 0xd8); + _mm_storeu_si128((__m128i*)dst_sobelx, _mm256_castsi256_si128(sum)); + src_y0 += 16; + src_y1 += 16; + src_y2 += 16; + dst_sobelx += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELXROW_AVX2 + +#ifdef HAS_SOBELYROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelYRow_AVX2(const uint8_t* src_y0, + const uint8_t* src_y1, + uint8_t* dst_sobely, + int width) { + while (width > 0) { + __m256i a0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y0)); + __m256i b0 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src_y1)); + __m256i a1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 1))); + __m256i b1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 1))); + __m256i a2 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y0 + 2))); + __m256i b2 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src_y1 + 2))); + __m256i d0 = _mm256_sub_epi16(a0, b0); + __m256i d1 = _mm256_sub_epi16(a1, b1); + __m256i d2 = _mm256_sub_epi16(a2, b2); + __m256i sum = + _mm256_add_epi16(_mm256_add_epi16(d0, d2), _mm256_add_epi16(d1, d1)); + sum = _mm256_abs_epi16(sum); + sum = _mm256_permute4x64_epi64(_mm256_packus_epi16(sum, sum), 0xd8); + _mm_storeu_si128((__m128i*)dst_sobely, _mm256_castsi256_si128(sum)); + src_y0 += 16; + src_y1 += 16; + dst_sobely += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELYROW_AVX2 + +#ifdef HAS_SOBELROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelRow_AVX2(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + const __m256i v_alpha = _mm256_set1_epi32((int)0xff000000u); + while (width > 0) { + __m128i vx = _mm_loadu_si128((const __m128i*)src_sobelx); + __m128i vy = _mm_loadu_si128((const __m128i*)src_sobely); + __m256i s16 = _mm256_cvtepu8_epi16(_mm_adds_epu8(vx, vy)); + s16 = _mm256_or_si256(s16, _mm256_slli_epi16(s16, 8)); + s16 = _mm256_permute4x64_epi64(s16, 0xd8); + __m256i p0 = _mm256_or_si256(_mm256_unpacklo_epi16(s16, s16), v_alpha); + __m256i p1 = _mm256_or_si256(_mm256_unpackhi_epi16(s16, s16), v_alpha); + _mm256_storeu_si256((__m256i*)dst_argb, p0); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), p1); + src_sobelx += 16; + src_sobely += 16; + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELROW_AVX2 + +#ifdef HAS_SOBELTOPLANEROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelToPlaneRow_AVX2(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width) { + while (width > 0) { + __m256i vx = _mm256_loadu_si256((const __m256i*)src_sobelx); + __m256i vy = _mm256_loadu_si256((const __m256i*)src_sobely); + _mm256_storeu_si256((__m256i*)dst_y, _mm256_adds_epu8(vx, vy)); + src_sobelx += 32; + src_sobely += 32; + dst_y += 32; + width -= 32; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELTOPLANEROW_AVX2 + +#ifdef HAS_SOBELXYROW_AVX2 +LIBYUV_TARGET_AVX2 +void SobelXYRow_AVX2(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + const __m256i v_ff00 = _mm256_set1_epi16((short)0xff00u); + while (width > 0) { + __m128i vx = _mm_loadu_si128((const __m128i*)src_sobelx); + __m128i vy = _mm_loadu_si128((const __m128i*)src_sobely); + __m128i vs = _mm_adds_epu8(vx, vy); + __m256i bg = + _mm256_or_si256(_mm256_cvtepu8_epi16(vy), + _mm256_slli_epi16(_mm256_cvtepu8_epi16(vs), 8)); + __m256i ra = _mm256_or_si256(_mm256_cvtepu8_epi16(vx), v_ff00); + bg = _mm256_permute4x64_epi64(bg, 0xd8); + ra = _mm256_permute4x64_epi64(ra, 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi16(bg, ra)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_unpackhi_epi16(bg, ra)); + src_sobelx += 16; + src_sobely += 16; + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_SOBELXYROW_AVX2 + +#ifdef HAS_ARGBAFFINEROW_AVX2 +LIBYUV_API +LIBYUV_TARGET_AVX2 +void ARGBAffineRow_AVX2(const uint8_t* src_argb, + int src_argb_stride, + uint8_t* dst_argb, + const float* src_dudv, + int width) { + static const float kRamp[8] = {0.0f, 1.0f, 2.0f, 3.0f, + 4.0f, 5.0f, 6.0f, 7.0f}; + float uv_tail[4] = {src_dudv[0], src_dudv[1], src_dudv[2], src_dudv[3]}; + if (width >= 8) { + const __m256 ramp = _mm256_loadu_ps(kRamp); + const __m256 step8 = _mm256_set1_ps(8.0f); + __m256 du = _mm256_set1_ps(uv_tail[2]); + __m256 dv = _mm256_set1_ps(uv_tail[3]); + __m256 u = + _mm256_add_ps(_mm256_set1_ps(uv_tail[0]), _mm256_mul_ps(ramp, du)); + __m256 v = + _mm256_add_ps(_mm256_set1_ps(uv_tail[1]), _mm256_mul_ps(ramp, dv)); + du = _mm256_mul_ps(du, step8); + dv = _mm256_mul_ps(dv, step8); + const __m256i v_stride = _mm256_set1_epi32(src_argb_stride); + while (width >= 8) { + __m256i iu = _mm256_cvttps_epi32(u); + __m256i iv = _mm256_cvttps_epi32(v); + __m256i idx = _mm256_add_epi32(_mm256_slli_epi32(iu, 2), + _mm256_mullo_epi32(iv, v_stride)); + __m256i pix = _mm256_i32gather_epi32((const int*)src_argb, idx, 1); + _mm256_storeu_si256((__m256i*)dst_argb, pix); + u = _mm256_add_ps(u, du); + v = _mm256_add_ps(v, dv); + dst_argb += 32; + width -= 8; + } + uv_tail[0] = _mm256_cvtss_f32(u); + uv_tail[1] = _mm256_cvtss_f32(v); + _mm256_zeroupper(); + } + if (width > 0) { + ARGBAffineRow_C(src_argb, src_argb_stride, dst_argb, uv_tail, width); + } +} +#endif // HAS_ARGBAFFINEROW_AVX2 + +#ifdef HAS_ARGBCOLORTABLEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + if (width >= 8) { + const __m256i m0 = _mm256_set1_epi32(0x000000ff); + const __m256i m1 = _mm256_set1_epi32(0x0000ff00); + const __m256i m2 = _mm256_set1_epi32(0x00ff0000); + const __m256i m3 = _mm256_set1_epi32((int)0xff000000u); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb); + __m256i c0 = + _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb, + _mm256_and_si256(v0, m0), 4), + m0); + __m256i c1 = _mm256_and_si256( + _mm256_i32gather_epi32((const int*)table_argb, + _mm256_and_si256(_mm256_srli_epi32(v0, 8), m0), + 4), + m1); + __m256i c2 = _mm256_and_si256( + _mm256_i32gather_epi32( + (const int*)table_argb, + _mm256_and_si256(_mm256_srli_epi32(v0, 16), m0), 4), + m2); + __m256i c3 = + _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb, + _mm256_srli_epi32(v0, 24), 4), + m3); + __m256i out = + _mm256_or_si256(_mm256_or_si256(c0, c1), _mm256_or_si256(c2, c3)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBColorTableRow_C(dst_argb, table_argb, width); + } +} +#endif // HAS_ARGBCOLORTABLEROW_AVX2 + +#ifdef HAS_RGBCOLORTABLEROW_AVX2 +LIBYUV_TARGET_AVX2 +void RGBColorTableRow_AVX2(uint8_t* dst_argb, + const uint8_t* table_argb, + int width) { + if (width >= 8) { + const __m256i m0 = _mm256_set1_epi32(0x000000ff); + const __m256i m1 = _mm256_set1_epi32(0x0000ff00); + const __m256i m2 = _mm256_set1_epi32(0x00ff0000); + const __m256i m3 = _mm256_set1_epi32((int)0xff000000u); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)dst_argb); + __m256i c0 = + _mm256_and_si256(_mm256_i32gather_epi32((const int*)table_argb, + _mm256_and_si256(v0, m0), 4), + m0); + __m256i c1 = _mm256_and_si256( + _mm256_i32gather_epi32((const int*)table_argb, + _mm256_and_si256(_mm256_srli_epi32(v0, 8), m0), + 4), + m1); + __m256i c2 = _mm256_and_si256( + _mm256_i32gather_epi32( + (const int*)table_argb, + _mm256_and_si256(_mm256_srli_epi32(v0, 16), m0), 4), + m2); + __m256i c3 = _mm256_and_si256(v0, m3); + __m256i out = + _mm256_or_si256(_mm256_or_si256(c0, c1), _mm256_or_si256(c2, c3)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + RGBColorTableRow_C(dst_argb, table_argb, width); + } +} +#endif // HAS_RGBCOLORTABLEROW_AVX2 + +#ifdef HAS_ARGBLUMACOLORTABLEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBLumaColorTableRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + const uint8_t* luma, + uint32_t lumacoeff) { + if (width >= 8) { + const __m256i v_coeff = _mm256_set1_epi32((int)(lumacoeff & 0x00ffffffu)); + const __m256i m_ff = _mm256_set1_epi32(0x000000ff); + const __m256i m_7f00 = _mm256_set1_epi32(0x00007f00); + const __m256i m_18 = _mm256_set1_epi32(0x00000018); + const __m256i m_alpha = _mm256_set1_epi32((int)0xff000000u); + while (width >= 8) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + __m256i y = _mm256_maddubs_epi16(v0, v_coeff); + y = _mm256_hadd_epi16(y, y); + y = _mm256_permute4x64_epi64(y, 0xd8); + y = _mm256_and_si256(_mm256_cvtepu16_epi32(_mm256_castsi256_si128(y)), + m_7f00); + + __m256i b_idx = + _mm256_srli_epi32(_mm256_or_si256(_mm256_and_si256(v0, m_ff), y), 2); + __m256i b_val = _mm256_i32gather_epi32((const int*)luma, b_idx, 4); + __m256i b_sh = _mm256_and_si256(_mm256_slli_epi32(v0, 3), m_18); + b_val = _mm256_and_si256(_mm256_srlv_epi32(b_val, b_sh), m_ff); + + __m256i g_idx = _mm256_srli_epi32( + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v0, 8), m_ff), y), + 2); + __m256i g_val = _mm256_i32gather_epi32((const int*)luma, g_idx, 4); + __m256i g_sh = _mm256_and_si256(_mm256_srli_epi32(v0, 5), m_18); + g_val = _mm256_slli_epi32( + _mm256_and_si256(_mm256_srlv_epi32(g_val, g_sh), m_ff), 8); + + __m256i r_idx = _mm256_srli_epi32( + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi32(v0, 16), m_ff), y), + 2); + __m256i r_val = _mm256_i32gather_epi32((const int*)luma, r_idx, 4); + __m256i r_sh = _mm256_and_si256(_mm256_srli_epi32(v0, 13), m_18); + r_val = _mm256_slli_epi32( + _mm256_and_si256(_mm256_srlv_epi32(r_val, r_sh), m_ff), 16); + + __m256i a_val = _mm256_and_si256(v0, m_alpha); + __m256i out = _mm256_or_si256(_mm256_or_si256(b_val, g_val), + _mm256_or_si256(r_val, a_val)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + ARGBLumaColorTableRow_C(src_argb, dst_argb, width, luma, lumacoeff); + } +} +#endif // HAS_ARGBLUMACOLORTABLEROW_AVX2 + +#ifdef HAS_BYTETOFLOATROW_AVX2 +LIBYUV_TARGET_AVX2 +void ByteToFloatRow_AVX2(const uint8_t* src, + float* dst, + float scale, + int width) { + const __m256 v_scale = _mm256_set1_ps(scale); + while (width > 0) { + __m256i i0 = _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)src)); + __m256i i1 = + _mm256_cvtepu8_epi32(_mm_loadl_epi64((const __m128i*)(src + 8))); + _mm256_storeu_ps(dst, _mm256_mul_ps(_mm256_cvtepi32_ps(i0), v_scale)); + _mm256_storeu_ps(dst + 8, _mm256_mul_ps(_mm256_cvtepi32_ps(i1), v_scale)); + src += 16; + dst += 16; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_BYTETOFLOATROW_AVX2 + +#ifdef HAS_GAUSSROW_AVX2 +LIBYUV_TARGET_AVX2 +void GaussRow_AVX2(const uint32_t* src, uint16_t* dst, int width) { + static const int8_t kShuf32To16[16] = {0, 1, 4, 5, 8, 9, 12, 13, + -1, -1, -1, -1, -1, -1, -1, -1}; + if (width >= 8) { + const __m256i v_128 = _mm256_set1_epi32(128); + const __m256i v_shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShuf32To16)); + while (width >= 8) { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src); + __m256i s1 = _mm256_loadu_si256((const __m256i*)(src + 1)); + __m256i s2 = _mm256_loadu_si256((const __m256i*)(src + 2)); + __m256i s3 = _mm256_loadu_si256((const __m256i*)(src + 3)); + __m256i s4 = _mm256_loadu_si256((const __m256i*)(src + 4)); + src += 8; + __m256i sum04 = _mm256_add_epi32(s0, s4); + __m256i sum13 = _mm256_slli_epi32(_mm256_add_epi32(s1, s3), 2); + __m256i sum2 = + _mm256_add_epi32(_mm256_slli_epi32(s2, 2), _mm256_slli_epi32(s2, 1)); + __m256i acc = + _mm256_srli_epi32(_mm256_add_epi32(_mm256_add_epi32(sum04, sum13), + _mm256_add_epi32(sum2, v_128)), + 8); + acc = _mm256_permute4x64_epi64(_mm256_shuffle_epi8(acc, v_shuf), 0xd8); + _mm_storeu_si128((__m128i*)dst, _mm256_castsi256_si128(acc)); + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + GaussRow_C(src, dst, width); + } +} +#endif // HAS_GAUSSROW_AVX2 + +#ifdef HAS_GAUSSROW_F32_AVX2 +LIBYUV_TARGET_AVX2 +void GaussRow_F32_AVX2(const float* src, float* dst, int width) { + if (width >= 8) { + const __m256 v4 = _mm256_set1_ps(4.0f); + const __m256 v6 = _mm256_set1_ps(6.0f); + const __m256 v_scale = _mm256_set1_ps(1.0f / 256.0f); + while (width >= 8) { + __m256 acc = _mm256_loadu_ps(src); + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 1), v4)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 2), v6)); + acc = _mm256_add_ps(acc, _mm256_mul_ps(_mm256_loadu_ps(src + 3), v4)); + acc = _mm256_add_ps(acc, _mm256_loadu_ps(src + 4)); + acc = _mm256_mul_ps(acc, v_scale); + _mm256_storeu_ps(dst, acc); + src += 8; + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + } + if (width > 0) { + GaussRow_F32_C(src, dst, width); + } +} +#endif // HAS_GAUSSROW_F32_AVX2 + +#ifdef HAS_SETROW_AVX2 +LIBYUV_TARGET_AVX2 +void SetRow_AVX2(uint8_t* dst, uint8_t v8, int width) { + const __m256i v = _mm256_set1_epi8((char)v8); + while (width >= 64) { + _mm256_storeu_si256((__m256i*)dst, v); + _mm256_storeu_si256((__m256i*)(dst + 32), v); + dst += 64; + width -= 64; + } + if (width > 0) { + _mm256_storeu_si256((__m256i*)dst, v); + } + _mm256_zeroupper(); +} +#endif // HAS_SETROW_AVX2 + +#ifdef HAS_ARGBSETROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBSetRow_AVX2(uint8_t* dst_argb, uint32_t v32, int width) { + const __m256i v = _mm256_set1_epi32((int)v32); + while (width >= 16) { + _mm256_storeu_si256((__m256i*)dst_argb, v); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), v); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + _mm256_storeu_si256((__m256i*)dst_argb, v); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBSETROW_AVX2 + +#ifdef HAS_COPYROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void CopyRow_16_AVX2(const uint16_t* src, uint16_t* dst, int width) { + while (width > 0) { + __m256i v0 = _mm256_loadu_si256((const __m256i*)src); + __m256i v1 = _mm256_loadu_si256((const __m256i*)(src + 16)); + _mm256_storeu_si256((__m256i*)dst, v0); + _mm256_storeu_si256((__m256i*)(dst + 16), v1); + src += 32; + dst += 32; + width -= 32; + } + _mm256_zeroupper(); +} +#endif // HAS_COPYROW_16_AVX2 + +#ifdef HAS_DETILEROW_AVX2 +LIBYUV_TARGET_AVX2 +void DetileRow_AVX2(const uint8_t* src, + ptrdiff_t src_tile_stride, + uint8_t* dst, + int width) { + while (width >= 32) { + __m128i t0 = _mm_loadu_si128((const __m128i*)src); + __m128i t1 = _mm_loadu_si128((const __m128i*)(src + src_tile_stride)); + src += 2 * src_tile_stride; + _mm256_storeu_si256((__m256i*)dst, _mm256_set_m128i(t1, t0)); + dst += 32; + width -= 32; + } + if (width > 0) { + _mm_storeu_si128((__m128i*)dst, _mm_loadu_si128((const __m128i*)src)); + } + _mm256_zeroupper(); +} +#endif // HAS_DETILEROW_AVX2 + +#ifdef HAS_DETILEROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void DetileRow_16_AVX2(const uint16_t* src, + ptrdiff_t src_tile_stride, + uint16_t* dst, + int width) { + while (width >= 32) { + __m256i t0 = _mm256_loadu_si256((const __m256i*)src); + __m256i t1 = _mm256_loadu_si256((const __m256i*)(src + src_tile_stride)); + src += 2 * src_tile_stride; + _mm256_storeu_si256((__m256i*)dst, t0); + _mm256_storeu_si256((__m256i*)(dst + 16), t1); + dst += 32; + width -= 32; + } + if (width > 0) { + _mm256_storeu_si256((__m256i*)dst, _mm256_loadu_si256((const __m256i*)src)); + } + _mm256_zeroupper(); +} +#endif // HAS_DETILEROW_16_AVX2 + +#ifdef HAS_DETILESPLITUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void DetileSplitUVRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_tile_stride, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kDeinterlaceUV_Win[16] = {0, 2, 4, 6, 8, 10, 12, 14, + 1, 3, 5, 7, 9, 11, 13, 15}; + const __m128i shuf128 = _mm_loadu_si128((const __m128i*)kDeinterlaceUV_Win); + const __m256i shuf256 = _mm256_broadcastsi128_si256(shuf128); + while (width >= 32) { + __m128i t0 = _mm_loadu_si128((const __m128i*)src_uv); + __m128i t1 = _mm_loadu_si128((const __m128i*)(src_uv + src_tile_stride)); + src_uv += 2 * src_tile_stride; + __m256i v = _mm256_permute4x64_epi64( + _mm256_shuffle_epi8(_mm256_set_m128i(t1, t0), shuf256), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(v)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_extracti128_si256(v, 1)); + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + __m128i v = + _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)src_uv), shuf128); + _mm_storel_epi64((__m128i*)dst_u, v); + _mm_storel_epi64((__m128i*)dst_v, _mm_unpackhi_epi64(v, v)); + } + _mm256_zeroupper(); +} +#endif // HAS_DETILESPLITUVROW_AVX2 + +#if defined(__x86_64__) || defined(_M_X64) + +#ifdef HAS_ARGB1555TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGB1555ToARGBRow_AVX512BW(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width) { + const __m512i zmm_f800 = _mm512_set1_epi16((short)0xf800); + const __m512i zmm_03e0 = _mm512_set1_epi16(0x03e0); + const __m512i zmm_ff00 = _mm512_set1_epi16((short)0xff00); + const __m512i zmm_0108 = _mm512_set1_epi16(0x0108); + const __m512i zmm_4200 = _mm512_set1_epi16(0x4200); + while (width >= 32) { + __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb1555); + src_argb1555 += 64; + __m512i r = _mm512_mulhi_epu16( + _mm512_and_si512(_mm512_slli_epi16(v0, 1), zmm_f800), zmm_0108); + __m512i b = _mm512_mulhi_epu16(_mm512_slli_epi16(v0, 11), zmm_0108); + __m512i rb = _mm512_or_si512(_mm512_slli_epi16(r, 8), b); + __m512i g = _mm512_mulhi_epu16(_mm512_and_si512(v0, zmm_03e0), zmm_4200); + __m512i a = _mm512_and_si512(_mm512_srai_epi16(v0, 8), zmm_ff00); + __m512i ga = _mm512_or_si512(g, a); + __m512i lo = _mm512_unpacklo_epi8(rb, ga); + __m512i hi = _mm512_unpackhi_epi8(rb, ga); + __m512i out0 = _mm512_shuffle_i64x2(lo, hi, 0x44); + __m512i out1 = _mm512_shuffle_i64x2(lo, hi, 0xee); + out0 = _mm512_shuffle_i64x2(out0, out0, 0xd8); + out1 = _mm512_shuffle_i64x2(out1, out1, 0xd8); + _mm512_storeu_si512((__m512i*)dst_argb, out0); + _mm512_storeu_si512((__m512i*)(dst_argb + 64), out1); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1ULL << width) - 1ULL); + __m512i v0 = _mm512_maskz_loadu_epi16(k, src_argb1555); + __m512i r = _mm512_mulhi_epu16( + _mm512_and_si512(_mm512_slli_epi16(v0, 1), zmm_f800), zmm_0108); + __m512i b = _mm512_mulhi_epu16(_mm512_slli_epi16(v0, 11), zmm_0108); + __m512i rb = _mm512_or_si512(_mm512_slli_epi16(r, 8), b); + __m512i g = _mm512_mulhi_epu16(_mm512_and_si512(v0, zmm_03e0), zmm_4200); + __m512i a = _mm512_and_si512(_mm512_srai_epi16(v0, 8), zmm_ff00); + __m512i ga = _mm512_or_si512(g, a); + __m512i lo = _mm512_unpacklo_epi8(rb, ga); + __m512i hi = _mm512_unpackhi_epi8(rb, ga); + __m512i out0 = _mm512_shuffle_i64x2(lo, hi, 0x44); + __m512i out1 = _mm512_shuffle_i64x2(lo, hi, 0xee); + out0 = _mm512_shuffle_i64x2(out0, out0, 0xd8); + out1 = _mm512_shuffle_i64x2(out1, out1, 0xd8); + if (width > 16) { + _mm512_storeu_si512((__m512i*)dst_argb, out0); + _mm512_mask_storeu_epi32(dst_argb + 64, (__mmask16)(k >> 16), out1); + } else { + _mm512_mask_storeu_epi32(dst_argb, (__mmask16)k, out0); + } + } + _mm256_zeroupper(); +} +#endif // HAS_ARGB1555TOARGBROW_AVX512BW + +#ifdef HAS_ARGBTORGB565DITHERROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToRGB565DitherRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width) { + __m128i d128 = _mm_cvtsi32_si128((int)dither4); + d128 = _mm_unpacklo_epi8(d128, d128); + d128 = _mm_unpacklo_epi16(d128, d128); + const __m512i zmm_dither = _mm512_broadcast_i32x4(d128); + const __m512i zmm_001f = _mm512_set1_epi32(0x0000001f); + const __m512i zmm_07e0 = _mm512_set1_epi32(0x000007e0); + const __m512i zmm_f800 = _mm512_set1_epi32(0x0000f800); + while (width >= 16) { + __m512i v0 = _mm512_adds_epu8(_mm512_loadu_si512((const __m512i*)src_argb), + zmm_dither); + src_argb += 64; + __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f); + __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0); + __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800); + __m256i rgb16 = + _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r)); + _mm256_storeu_si256((__m256i*)dst_rgb, rgb16); + dst_rgb += 32; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v0 = + _mm512_adds_epu8(_mm512_maskz_loadu_epi32(k, src_argb), zmm_dither); + __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f); + __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0); + __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800); + __m256i rgb16 = + _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r)); + _mm256_mask_storeu_epi16(dst_rgb, k, rgb16); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTORGB565DITHERROW_AVX512BW + +#ifdef HAS_ARGBTORGB565ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToRGB565Row_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + const __m512i zmm_001f = _mm512_set1_epi32(0x0000001f); + const __m512i zmm_07e0 = _mm512_set1_epi32(0x000007e0); + const __m512i zmm_f800 = _mm512_set1_epi32(0x0000f800); + while (width >= 16) { + __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f); + __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0); + __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800); + __m256i rgb16 = + _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r)); + _mm256_storeu_si256((__m256i*)dst_rgb, rgb16); + dst_rgb += 32; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v0 = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i b = _mm512_and_si512(_mm512_srli_epi32(v0, 3), zmm_001f); + __m512i g = _mm512_and_si512(_mm512_srli_epi32(v0, 5), zmm_07e0); + __m512i r = _mm512_and_si512(_mm512_srli_epi32(v0, 8), zmm_f800); + __m256i rgb16 = + _mm512_cvtepi32_epi16(_mm512_or_si512(_mm512_or_si512(b, g), r)); + _mm256_mask_storeu_epi16(dst_rgb, k, rgb16); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTORGB565ROW_AVX512BW + +#ifdef HAS_MERGEUVROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW +void MergeUVRow_16_AVX512BW(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + const __m128i sh_u = _mm_cvtsi32_si128(16 - depth); + const __m128i sh_v = _mm_cvtsi32_si128(32 - depth); + while (width >= 16) { + __m512i u32 = + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_u)); + __m512i v32 = + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src_v)); + src_u += 16; + src_v += 16; + u32 = _mm512_sll_epi16(u32, sh_u); + v32 = _mm512_sll_epi32(v32, sh_v); + _mm512_storeu_si512((__m512i*)dst_uv, _mm512_or_si512(u32, v32)); + dst_uv += 32; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i u32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_u)); + __m512i v32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src_v)); + u32 = _mm512_sll_epi16(u32, sh_u); + v32 = _mm512_sll_epi32(v32, sh_v); + _mm512_mask_storeu_epi32(dst_uv, k, _mm512_or_si512(u32, v32)); + } + _mm256_zeroupper(); +} +#endif // HAS_MERGEUVROW_16_AVX512BW + +#ifdef HAS_DIVIDEROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW +void DivideRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + while (width >= 32) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_y); + src_y += 32; + _mm512_storeu_si512((__m512i*)dst_y, _mm512_mulhi_epu16(v, zmm_scale)); + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi16(k, src_y); + _mm512_mask_storeu_epi16(dst_y, k, _mm512_mulhi_epu16(v, zmm_scale)); + } + _mm256_zeroupper(); +} +#endif // HAS_DIVIDEROW_16_AVX512BW + +#ifdef HAS_YUY2TOYROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void YUY2ToYRow_AVX512BW(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + while (width >= 32) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_yuy2); + src_yuy2 += 64; + _mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(v)); + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi16(k, src_yuy2); + _mm256_mask_storeu_epi8(dst_y, k, _mm512_cvtepi16_epi8(v)); + } + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOYROW_AVX512BW + +#ifdef HAS_YUY2TOUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void YUY2ToUVRow_AVX512BW(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const ptrdiff_t stride = (ptrdiff_t)stride_yuy2; + while (width >= 32) { + __m512i r0 = _mm512_loadu_si512((const __m512i*)src_yuy2); + __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_yuy2 + stride)); + src_yuy2 += 64; + __m512i avg = _mm512_avg_epu8(r0, r1); + _mm_storeu_si128((__m128i*)dst_u, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 8))); + _mm_storeu_si128((__m128i*)dst_v, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 24))); + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i r0 = _mm512_maskz_loadu_epi32(k, src_yuy2); + __m512i r1 = _mm512_maskz_loadu_epi32(k, src_yuy2 + stride); + __m512i avg = _mm512_avg_epu8(r0, r1); + _mm_mask_storeu_epi8(dst_u, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 8))); + _mm_mask_storeu_epi8(dst_v, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 24))); + } + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOUVROW_AVX512BW + +#ifdef HAS_UYVYTOYROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void UYVYToYRow_AVX512BW(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + while (width >= 32) { + __m512i v = _mm512_loadu_si512((const __m512i*)src_uyvy); + src_uyvy += 64; + _mm256_storeu_si256((__m256i*)dst_y, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8))); + dst_y += 32; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i v = _mm512_maskz_loadu_epi16(k, src_uyvy); + _mm256_mask_storeu_epi8(dst_y, k, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(v, 8))); + } + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOYROW_AVX512BW + +#ifdef HAS_UYVYTOUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void UYVYToUVRow_AVX512BW(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const ptrdiff_t stride = (ptrdiff_t)stride_uyvy; + while (width >= 32) { + __m512i r0 = _mm512_loadu_si512((const __m512i*)src_uyvy); + __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uyvy + stride)); + src_uyvy += 64; + __m512i avg = _mm512_avg_epu8(r0, r1); + _mm_storeu_si128((__m128i*)dst_u, _mm512_cvtepi32_epi8(avg)); + _mm_storeu_si128((__m128i*)dst_v, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 16))); + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i r0 = _mm512_maskz_loadu_epi32(k, src_uyvy); + __m512i r1 = _mm512_maskz_loadu_epi32(k, src_uyvy + stride); + __m512i avg = _mm512_avg_epu8(r0, r1); + _mm_mask_storeu_epi8(dst_u, k, _mm512_cvtepi32_epi8(avg)); + _mm_mask_storeu_epi8(dst_v, k, + _mm512_cvtepi32_epi8(_mm512_srli_epi32(avg, 16))); + } + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOUVROW_AVX512BW + +#ifdef HAS_ARGBSEPIAROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI +void ARGBSepiaRow_AVX512VNNI(uint8_t* dst_argb, int width) { + const __m512i zero = _mm512_setzero_si512(); + const __m512i c_b = _mm512_set1_epi32(0x00234411); + const __m512i c_g = _mm512_set1_epi32(0x002d5816); + const __m512i c_r = _mm512_set1_epi32(0x00326218); + const __m512i max255 = _mm512_set1_epi32(255); + const __m512i a_mask = _mm512_set1_epi32((int)0xff000000u); + while (width >= 16) { + __m512i v0 = _mm512_loadu_si512((const __m512i*)dst_argb); + __m512i b = _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_b), 7); + __m512i g = _mm512_min_epu32( + _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_g), 7), max255); + __m512i r = _mm512_min_epu32( + _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_r), 7), max255); + __m512i a = _mm512_and_si512(v0, a_mask); + __m512i out = _mm512_or_si512(_mm512_or_si512(b, _mm512_slli_epi32(g, 8)), + _mm512_or_si512(_mm512_slli_epi32(r, 16), a)); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v0 = _mm512_maskz_loadu_epi32(k, dst_argb); + __m512i b = _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_b), 7); + __m512i g = _mm512_min_epu32( + _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_g), 7), max255); + __m512i r = _mm512_min_epu32( + _mm512_srli_epi32(_mm512_dpbusd_epi32(zero, v0, c_r), 7), max255); + __m512i a = _mm512_and_si512(v0, a_mask); + __m512i out = _mm512_or_si512(_mm512_or_si512(b, _mm512_slli_epi32(g, 8)), + _mm512_or_si512(_mm512_slli_epi32(r, 16), a)); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBSEPIAROW_AVX512VNNI + +#ifdef HAS_ARGBCOLORMATRIXROW_AVX512VNNI +LIBYUV_TARGET_AVX512VNNI +void ARGBColorMatrixRow_AVX512VNNI(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + const __m512i zero = _mm512_setzero_si512(); + const __m512i max255 = _mm512_set1_epi32(255); + const __m128i m128 = _mm_loadu_si128((const __m128i*)matrix_argb); + const __m512i m0 = _mm512_set1_epi32(_mm_cvtsi128_si32(m128)); + const __m512i m1 = + _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 4))); + const __m512i m2 = + _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 8))); + const __m512i m3 = + _mm512_set1_epi32(_mm_cvtsi128_si32(_mm_srli_si128(m128, 12))); + while (width >= 16) { + __m512i v0 = _mm512_loadu_si512((const __m512i*)src_argb); + src_argb += 64; + __m512i b = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m0), 6), zero), + max255); + __m512i g = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m1), 6), zero), + max255); + __m512i r = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m2), 6), zero), + max255); + __m512i a = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m3), 6), zero), + max255); + __m512i out = _mm512_or_si512( + _mm512_or_si512(b, _mm512_slli_epi32(g, 8)), + _mm512_or_si512(_mm512_slli_epi32(r, 16), _mm512_slli_epi32(a, 24))); + _mm512_storeu_si512((__m512i*)dst_argb, out); + dst_argb += 64; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i v0 = _mm512_maskz_loadu_epi32(k, src_argb); + __m512i b = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m0), 6), zero), + max255); + __m512i g = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m1), 6), zero), + max255); + __m512i r = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m2), 6), zero), + max255); + __m512i a = _mm512_min_epu32( + _mm512_max_epi32( + _mm512_srai_epi32(_mm512_dpbusd_epi32(zero, v0, m3), 6), zero), + max255); + __m512i out = _mm512_or_si512( + _mm512_or_si512(b, _mm512_slli_epi32(g, 8)), + _mm512_or_si512(_mm512_slli_epi32(r, 16), _mm512_slli_epi32(a, 24))); + _mm512_mask_storeu_epi32(dst_argb, k, out); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBCOLORMATRIXROW_AVX512VNNI + +#ifdef HAS_HALFFLOATROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void HalfFloatRow_AVX512BW(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + const __m512 vscale = _mm512_set1_ps(scale); + const __m512 vmult = _mm512_set1_ps(1.9259299444e-34f); + while (width >= 16) { + __m512i s32 = + _mm512_cvtepu16_epi32(_mm256_loadu_si256((const __m256i*)src)); + src += 16; + __m512 f = + _mm512_mul_ps(_mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale), vmult); + __m256i h16 = + _mm512_cvtepi32_epi16(_mm512_srli_epi32(_mm512_castps_si512(f), 13)); + _mm256_storeu_si256((__m256i*)dst, h16); + dst += 16; + width -= 16; + } + if (width > 0) { + __mmask16 k = (__mmask16)((1u << width) - 1u); + __m512i s32 = _mm512_cvtepu16_epi32(_mm256_maskz_loadu_epi16(k, src)); + __m512 f = + _mm512_mul_ps(_mm512_mul_ps(_mm512_cvtepi32_ps(s32), vscale), vmult); + __m256i h16 = + _mm512_cvtepi32_epi16(_mm512_srli_epi32(_mm512_castps_si512(f), 13)); + _mm256_mask_storeu_epi16(dst, k, h16); + } + _mm256_zeroupper(); +} +#endif // HAS_HALFFLOATROW_AVX512BW + +#ifdef HAS_CONVERT8TO8ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void Convert8To8Row_AVX512BW(const uint8_t* src_y, + uint8_t* dst_y, + int scale, + int bias, + int width) { + const __m512i zmm_scale = + _mm512_slli_epi16(_mm512_set1_epi16((short)scale), 8); + const __m512i zmm_bias = _mm512_set1_epi8((char)bias); + const __m512i zmm_zero = _mm512_setzero_si512(); + while (width >= 64) { + __m512i y8 = _mm512_loadu_si512((const __m512i*)src_y); + src_y += 64; + __m512i y_lo = + _mm512_mulhi_epu16(_mm512_unpacklo_epi8(y8, zmm_zero), zmm_scale); + __m512i y_hi = + _mm512_mulhi_epu16(_mm512_unpackhi_epi8(y8, zmm_zero), zmm_scale); + __m512i out8 = _mm512_add_epi8(_mm512_packus_epi16(y_lo, y_hi), zmm_bias); + _mm512_storeu_si512((__m512i*)dst_y, out8); + dst_y += 64; + width -= 64; + } + if (width > 0) { + __mmask64 k = (__mmask64)((1ULL << width) - 1ULL); + __m512i y8 = _mm512_maskz_loadu_epi8(k, src_y); + __m512i y_lo = + _mm512_mulhi_epu16(_mm512_unpacklo_epi8(y8, zmm_zero), zmm_scale); + __m512i y_hi = + _mm512_mulhi_epu16(_mm512_unpackhi_epi8(y8, zmm_zero), zmm_scale); + __m512i out8 = _mm512_add_epi8(_mm512_packus_epi16(y_lo, y_hi), zmm_bias); + _mm512_mask_storeu_epi8(dst_y, k, out8); + } + _mm256_zeroupper(); +} +#endif // HAS_CONVERT8TO8ROW_AVX512BW + +// =========================================================================== +// Extended YUV (I422, I444, YUY2, UYVY, I210, I212, I410, P210, P410) +// to ARGB / RGBA / RGB24 / AR30 (AVX512BW / AVX512VBMI) +// =========================================================================== + +static const uint16_t kSplitQuadWords_ExtWin[32] = { + 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, + 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15}; +static const uint64_t kUnpermuteAVX512_ExtWin[8] = {0, 4, 1, 5, 2, 6, 3, 7}; +static const uint16_t kPermAR30_0_Win[32] = { + 0, 32, 1, 33, 2, 34, 3, 35, 4, 36, 5, 37, 6, 38, 7, 39, + 8, 40, 9, 41, 10, 42, 11, 43, 12, 44, 13, 45, 14, 46, 15, 47}; +static const uint16_t kPermAR30_1_Win[32] = { + 16, 48, 17, 49, 18, 50, 19, 51, 20, 52, 21, 53, 22, 54, 23, 55, + 24, 56, 25, 57, 26, 58, 27, 59, 28, 60, 29, 61, 30, 62, 31, 63}; +static const uint8_t kShuffleYUY2Y_Win[16] = {0, 0, 2, 2, 4, 4, 6, 6, + 8, 8, 10, 10, 12, 12, 14, 14}; +static const uint8_t kShuffleYUY2UV_Win[16] = {1, 3, 1, 3, 5, 7, 5, 7, + 9, 11, 9, 11, 13, 15, 13, 15}; +static const uint8_t kShuffleUYVYY_Win[16] = {1, 1, 3, 3, 5, 5, 7, 7, + 9, 9, 11, 11, 13, 13, 15, 15}; +static const uint8_t kShuffleUYVYUV_Win[16] = {0, 2, 0, 2, 4, 6, 4, 6, + 8, 10, 8, 10, 12, 14, 12, 14}; + +#define YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants) \ + const __m512i zmm_kUVToB = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToB)); \ + const __m512i zmm_kUVToG = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToG)); \ + const __m512i zmm_kUVToR = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kUVToR)); \ + const __m512i zmm_kYToRgb = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kYToRgb)); \ + const __m512i zmm_kYBiasToRgb = _mm512_broadcast_i32x4( \ + _mm_loadu_si128((const __m128i*)(yuvconstants)->kYBiasToRgb)); \ + const __m512i zmm_128 = _mm512_set1_epi8((char)0x80) + +#define YUVTORGB16_WIN_AVX512BW \ + zmm3 = _mm512_sub_epi8(zmm3, zmm_128); \ + zmm4 = _mm512_add_epi16(_mm512_mulhi_epu16(zmm4, zmm_kYToRgb), \ + zmm_kYBiasToRgb); \ + __m512i b16 = \ + _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToB, zmm3), zmm4); \ + __m512i g16 = \ + _mm512_subs_epi16(zmm4, _mm512_maddubs_epi16(zmm_kUVToG, zmm3)); \ + __m512i r16 = _mm512_adds_epi16(_mm512_maddubs_epi16(zmm_kUVToR, zmm3), zmm4) + +#define YUVTORGB8_WIN_AVX512BW \ + YUVTORGB16_WIN_AVX512BW; \ + __m512i zmm0 = _mm512_packus_epi16(_mm512_srai_epi16(b16, 6), zmm5); \ + __m512i zmm1 = _mm512_packus_epi16(_mm512_srai_epi16(g16, 6), zmm5); \ + __m512i zmm2 = _mm512_packus_epi16(_mm512_srai_epi16(r16, 6), zmm5) + +#define STOREARGB_WIN_AVX512BW(dst_ptr) \ + do { \ + __m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \ + _mm512_unpacklo_epi8(zmm0, zmm1)); \ + __m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \ + _mm512_unpacklo_epi8(zmm2, zmm5)); \ + _mm512_storeu_si512((__m512i*)(dst_ptr), _mm512_unpacklo_epi16(bg, ra)); \ + _mm512_storeu_si512((__m512i*)((dst_ptr) + 64), \ + _mm512_unpackhi_epi16(bg, ra)); \ + } while (0) + +#define STOREARGB_MASK_WIN_AVX512BW(dst_ptr, m32) \ + do { \ + __m512i bg = _mm512_permutexvar_epi64(zmm_unperm, \ + _mm512_unpacklo_epi8(zmm0, zmm1)); \ + __m512i ra = _mm512_permutexvar_epi64(zmm_unperm, \ + _mm512_unpacklo_epi8(zmm2, zmm5)); \ + _mm512_mask_storeu_epi32((dst_ptr), (__mmask16)(m32), \ + _mm512_unpacklo_epi16(bg, ra)); \ + _mm512_mask_storeu_epi32((dst_ptr) + 64, (__mmask16)((m32) >> 16), \ + _mm512_unpackhi_epi16(bg, ra)); \ + } while (0) + +#define PACKAR30_WIN_AVX512BW(dst0, dst1) \ + b16 = _mm512_max_epi16(_mm512_min_epi16(b16, zmm_3ff0), zmm_zero); \ + g16 = _mm512_max_epi16(_mm512_min_epi16(g16, zmm_3ff0), zmm_zero); \ + r16 = _mm512_max_epi16(_mm512_min_epi16(r16, zmm_3ff0), zmm_zero); \ + __m512i lo16 = \ + _mm512_or_si512(_mm512_srli_epi16(b16, 4), \ + _mm512_and_si512(_mm512_slli_epi16(g16, 6), zmm_fc00)); \ + __m512i hi16 = \ + _mm512_or_si512(_mm512_or_si512(_mm512_and_si512(r16, zmm_3ff0), \ + _mm512_srli_epi16(g16, 10)), \ + zmm_c000); \ + __m512i dst0 = _mm512_permutex2var_epi16(lo16, zmm_perm0, hi16); \ + __m512i dst1 = _mm512_permutex2var_epi16(lo16, zmm_perm1, hi16) + +#ifdef HAS_I422TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I422ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m128i xu = _mm_loadu_si128((const __m128i*)u_buf); + __m128i xv = _mm_loadu_si128((const __m128i*)v_buf); + u_buf += 16; + v_buf += 16; + __m256i uv256 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + y_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf); + __m128i xv = _mm_maskz_loadu_epi8(k_uv, v_buf); + __m256i uv256 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I422TOARGBROW_AVX512BW + +#ifdef HAS_I422TORGBAROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I422ToRGBARow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgba, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m128i xu = _mm_loadu_si128((const __m128i*)u_buf); + __m128i xv = _mm_loadu_si128((const __m128i*)v_buf); + u_buf += 16; + v_buf += 16; + __m256i uv256 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + y_buf += 32; + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + __m512i ab = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); + __m512i gr = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); + _mm512_storeu_si512((__m512i*)dst_rgba, _mm512_unpacklo_epi16(ab, gr)); + _mm512_storeu_si512((__m512i*)(dst_rgba + 64), + _mm512_unpackhi_epi16(ab, gr)); + dst_rgba += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m128i xu = _mm_maskz_loadu_epi8(k_uv, u_buf); + __m128i xv = _mm_maskz_loadu_epi8(k_uv, v_buf); + __m256i uv256 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_unpacklo_epi8(xu, xv)), + _mm_unpackhi_epi8(xu, xv), 1); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8((__mmask32)m32, y_buf)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + __m512i ab = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm5, zmm0)); + __m512i gr = + _mm512_permutexvar_epi64(zmm_unperm, _mm512_unpacklo_epi8(zmm1, zmm2)); + _mm512_mask_storeu_epi32(dst_rgba, (__mmask16)m32, + _mm512_unpacklo_epi16(ab, gr)); + _mm512_mask_storeu_epi32(dst_rgba + 64, (__mmask16)(m32 >> 16), + _mm512_unpackhi_epi16(ab, gr)); + } + _mm256_zeroupper(); +} +#endif // HAS_I422TORGBAROW_AVX512BW + +#ifdef HAS_I444TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I444ToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i u16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf)); + __m512i v16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + u_buf += 32; + v_buf += 32; + y_buf += 32; + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf)); + __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf)); + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I444TOARGBROW_AVX512BW + +#ifdef HAS_I444ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I444AlphaToARGBRow_AVX512BW(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + const uint8_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + while (width >= 32) { + __m512i u16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf)); + __m512i v16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + __m512i a16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)a_buf)); + u_buf += 32; + v_buf += 32; + y_buf += 32; + a_buf += 32; + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf)); + __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf)); + __m512i a16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, a_buf)); + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I444ALPHATOARGBROW_AVX512BW + +#ifdef HAS_I444TORGB24ROW_AVX512VBMI +static const uint8_t kMaskBG_I444[64] = { + 0x00, 0x40, 0x01, 0x41, 0x02, 0x42, 0x03, 0x43, 0x04, 0x44, 0x05, + 0x45, 0x06, 0x46, 0x07, 0x47, 0x10, 0x50, 0x11, 0x51, 0x12, 0x52, + 0x13, 0x53, 0x14, 0x54, 0x15, 0x55, 0x16, 0x56, 0x17, 0x57, 0x20, + 0x60, 0x21, 0x61, 0x22, 0x62, 0x23, 0x63, 0x24, 0x64, 0x25, 0x65, + 0x26, 0x66, 0x27, 0x67, 0x30, 0x70, 0x31, 0x71, 0x32, 0x72, 0x33, + 0x73, 0x34, 0x74, 0x35, 0x75, 0x36, 0x76, 0x37, 0x77}; +static const uint8_t kMaskDST0_I444[64] = { + 0x00, 0x01, 0x40, 0x02, 0x03, 0x41, 0x04, 0x05, 0x42, 0x06, 0x07, + 0x43, 0x08, 0x09, 0x44, 0x0a, 0x0b, 0x45, 0x0c, 0x0d, 0x46, 0x0e, + 0x0f, 0x47, 0x10, 0x11, 0x50, 0x12, 0x13, 0x51, 0x14, 0x15, 0x52, + 0x16, 0x17, 0x53, 0x18, 0x19, 0x54, 0x1a, 0x1b, 0x55, 0x1c, 0x1d, + 0x56, 0x1e, 0x1f, 0x57, 0x20, 0x21, 0x60, 0x22, 0x23, 0x61, 0x24, + 0x25, 0x62, 0x26, 0x27, 0x63, 0x28, 0x29, 0x64, 0x2a}; +static const uint8_t kMaskDST1_I444[64] = { + 0x2b, 0x65, 0x2c, 0x2d, 0x66, 0x2e, 0x2f, 0x67, 0x30, 0x31, 0x70, + 0x32, 0x33, 0x71, 0x34, 0x35, 0x72, 0x36, 0x37, 0x73, 0x38, 0x39, + 0x74, 0x3a, 0x3b, 0x75, 0x3c, 0x3d, 0x76, 0x3e, 0x3f, 0x77, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; + +LIBYUV_TARGET_AVX512VBMI +void I444ToRGB24Row_AVX512VBMI(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_rgb24, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_mask_BG = _mm512_loadu_si512((const __m512i*)kMaskBG_I444); + const __m512i zmm_mask_DST0 = + _mm512_loadu_si512((const __m512i*)kMaskDST0_I444); + const __m512i zmm_mask_DST1 = + _mm512_loadu_si512((const __m512i*)kMaskDST1_I444); + while (width >= 32) { + __m512i u16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)u_buf)); + __m512i v16 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)v_buf)); + __m512i zmm4 = + _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)y_buf)); + u_buf += 32; + v_buf += 32; + y_buf += 32; + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB16_WIN_AVX512BW; + __m512i zmm0 = _mm512_srai_epi16(b16, 6); + __m512i zmm1 = _mm512_srai_epi16(g16, 6); + __m512i zmm2 = _mm512_srai_epi16(r16, 6); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + _mm512_storeu_si512((__m512i*)dst_rgb24, zmm_dst0); + _mm256_storeu_si256((__m256i*)(dst_rgb24 + 64), + _mm512_castsi512_si256(zmm_dst1)); + dst_rgb24 += 96; + width -= 32; + } + if (width > 0) { + __mmask32 k = (__mmask32)((1u << width) - 1u); + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, u_buf)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, v_buf)); + __m512i zmm4 = _mm512_cvtepu8_epi16(_mm256_maskz_loadu_epi8(k, y_buf)); + __m512i zmm3 = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + zmm4 = _mm512_or_si512(zmm4, _mm512_slli_epi16(zmm4, 8)); + YUVTORGB16_WIN_AVX512BW; + __m512i zmm0 = _mm512_srai_epi16(b16, 6); + __m512i zmm1 = _mm512_srai_epi16(g16, 6); + __m512i zmm2 = _mm512_srai_epi16(r16, 6); + zmm0 = _mm512_packus_epi16(zmm0, zmm0); + zmm1 = _mm512_packus_epi16(zmm1, zmm1); + zmm2 = _mm512_packus_epi16(zmm2, zmm2); + __m512i zmm_BG = _mm512_permutex2var_epi8(zmm0, zmm_mask_BG, zmm1); + __m512i zmm_dst0 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST0, zmm2); + __m512i zmm_dst1 = _mm512_permutex2var_epi8(zmm_BG, zmm_mask_DST1, zmm2); + int dst_bytes = width * 3; + __mmask64 k0 = dst_bytes >= 64 ? ~0ull : (1ull << dst_bytes) - 1ull; + __mmask32 k1 = dst_bytes > 64 ? (1u << (dst_bytes - 64)) - 1u : 0u; + _mm512_mask_storeu_epi8(dst_rgb24, k0, zmm_dst0); + _mm256_mask_storeu_epi8(dst_rgb24 + 64, k1, + _mm512_castsi512_si256(zmm_dst1)); + } + _mm256_zeroupper(); +} +#endif // HAS_I444TORGB24ROW_AVX512VBMI + +#ifdef HAS_YUY2TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void YUY2ToARGBRow_AVX512BW(const uint8_t* yuy2_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm_shuf_y = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleYUY2Y_Win)); + const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win)); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i in = _mm512_loadu_si512((const __m512i*)yuy2_buf); + yuy2_buf += 64; + __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y); + __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k2 = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i in = _mm512_maskz_loadu_epi32(k2, yuy2_buf); + __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y); + __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOARGBROW_AVX512BW + +#ifdef HAS_UYVYTOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void UYVYToARGBRow_AVX512BW(const uint8_t* uyvy_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm_shuf_y = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleUYVYY_Win)); + const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleUYVYUV_Win)); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i in = _mm512_loadu_si512((const __m512i*)uyvy_buf); + uyvy_buf += 64; + __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y); + __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k2 = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i in = _mm512_maskz_loadu_epi32(k2, uyvy_buf); + __m512i zmm4 = _mm512_shuffle_epi8(in, zmm_shuf_y); + __m512i zmm3 = _mm512_shuffle_epi8(in, zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOARGBROW_AVX512BW + +#ifdef HAS_I210TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m256i u_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2); + __m256i v_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2); + __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I210TOARGBROW_AVX512BW + +#ifdef HAS_I210TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m256i u_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2); + __m256i v_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2); + __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_I210TOAR30ROW_AVX512BW + +#ifdef HAS_I210ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I210AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + while (width >= 32) { + __m256i u_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 2); + __m256i v_s = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i a16 = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)a_buf), 2); + u_buf += 16; + v_buf += 16; + y_buf += 32; + a_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 2); + __m256i v_s = _mm256_srli_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i a16 = + _mm512_srli_epi16(_mm512_maskz_loadu_epi16((__mmask32)m32, a_buf), 2); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I210ALPHATOARGBROW_AVX512BW + +#ifdef HAS_I212TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I212ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m256i u_s = + _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 4); + __m256i v_s = + _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 4); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 4); + __m256i v_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 4); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I212TOARGBROW_AVX512BW + +#ifdef HAS_I212TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I212ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_quad = + _mm512_loadu_si512((const __m512i*)kSplitQuadWords_ExtWin); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m256i u_s = + _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)u_buf), 4); + __m256i v_s = + _mm256_srai_epi16(_mm256_loadu_si256((const __m256i*)v_buf), 4); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 32; + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m256i u_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, u_buf), 4); + __m256i v_s = _mm256_srai_epi16(_mm256_maskz_loadu_epi16(k_uv, v_buf), 4); + __m512i y16 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m256i uv256 = _mm256_unpacklo_epi8(_mm256_packus_epi16(u_s, u_s), + _mm256_packus_epi16(v_s, v_s)); + __m512i zmm3 = + _mm512_permutexvar_epi16(zmm_quad, _mm512_zextsi256_si512(uv256)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 4), _mm512_srli_epi16(y16, 8)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_I212TOAR30ROW_AVX512BW + +#ifdef HAS_I410TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i u_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2); + __m512i v_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 32; + v_buf += 32; + y_buf += 32; + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2); + __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf); + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I410TOARGBROW_AVX512BW + +#ifdef HAS_I410TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m512i u_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2); + __m512i v_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + u_buf += 32; + v_buf += 32; + y_buf += 32; + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2); + __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf); + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_I410TOAR30ROW_AVX512BW + +#ifdef HAS_I410ALPHATOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void I410AlphaToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* u_buf, + const uint16_t* v_buf, + const uint16_t* a_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + while (width >= 32) { + __m512i u_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)u_buf), 2); + __m512i v_s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)v_buf), 2); + __m512i y16 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i a16 = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)a_buf), 2); + u_buf += 32; + v_buf += 32; + y_buf += 32; + a_buf += 32; + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask32 k = (__mmask32)m32; + __m512i u_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, u_buf), 2); + __m512i v_s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, v_buf), 2); + __m512i y16 = _mm512_maskz_loadu_epi16(k, y_buf); + __m512i a16 = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, a_buf), 2); + __m512i zmm3 = _mm512_unpacklo_epi8(_mm512_packus_epi16(u_s, u_s), + _mm512_packus_epi16(v_s, v_s)); + __m512i zmm4 = + _mm512_or_si512(_mm512_slli_epi16(y16, 6), _mm512_srli_epi16(y16, 4)); + __m512i zmm5 = _mm512_packus_epi16(a16, a16); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_I410ALPHATOARGBROW_AVX512BW + +#ifdef HAS_P210TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void P210ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win)); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i zmm3 = _mm512_shuffle_epi8( + _mm512_loadu_si512((const __m512i*)uv_buf), zmm_shuf_uv); + y_buf += 32; + uv_buf += 32; + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i zmm3 = _mm512_shuffle_epi8(_mm512_maskz_loadu_epi32(k_uv, uv_buf), + zmm_shuf_uv); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_P210TOARGBROW_AVX512BW + +#ifdef HAS_P210TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void P210ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_shuf_uv = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleYUY2UV_Win)); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i zmm3 = _mm512_shuffle_epi8( + _mm512_loadu_si512((const __m512i*)uv_buf), zmm_shuf_uv); + y_buf += 32; + uv_buf += 32; + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __mmask16 k_uv = (__mmask16)((1u << ((width + 1) >> 1)) - 1u); + __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i zmm3 = _mm512_shuffle_epi8(_mm512_maskz_loadu_epi32(k_uv, uv_buf), + zmm_shuf_uv); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_P210TOAR30ROW_AVX512BW + +#ifdef HAS_P410TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void P410ToARGBRow_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_unperm = + _mm512_loadu_si512((const __m512i*)kUnpermuteAVX512_ExtWin); + const __m512i zmm5 = _mm512_set1_epi32(-1); + while (width >= 32) { + __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i uv0 = _mm512_loadu_si512((const __m512i*)uv_buf); + __m512i uv1 = _mm512_loadu_si512((const __m512i*)(uv_buf + 32)); + y_buf += 32; + uv_buf += 64; + __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)); + __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)); + __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_WIN_AVX512BW(dst_argb); + dst_argb += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i uv0 = _mm512_maskz_loadu_epi32((__mmask16)m32, uv_buf); + __m512i uv1 = _mm512_maskz_loadu_epi32((__mmask16)(m32 >> 16), uv_buf + 32); + __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)); + __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)); + __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1); + YUVTORGB8_WIN_AVX512BW; + STOREARGB_MASK_WIN_AVX512BW(dst_argb, m32); + } + _mm256_zeroupper(); +} +#endif // HAS_P410TOARGBROW_AVX512BW + +#ifdef HAS_P410TOAR30ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void P410ToAR30Row_AVX512BW(const uint16_t* y_buf, + const uint16_t* uv_buf, + uint8_t* dst_ar30, + const struct YuvConstants* yuvconstants, + int width) { + YUVTORGB_SETUP_WIN_AVX512BW(yuvconstants); + const __m512i zmm_perm0 = _mm512_loadu_si512((const __m512i*)kPermAR30_0_Win); + const __m512i zmm_perm1 = _mm512_loadu_si512((const __m512i*)kPermAR30_1_Win); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_3ff0 = _mm512_set1_epi16(0x3ff0); + const __m512i zmm_fc00 = _mm512_set1_epi16((short)0xfc00); + const __m512i zmm_c000 = _mm512_set1_epi16((short)0xc000); + while (width >= 32) { + __m512i zmm4 = _mm512_loadu_si512((const __m512i*)y_buf); + __m512i uv0 = _mm512_loadu_si512((const __m512i*)uv_buf); + __m512i uv1 = _mm512_loadu_si512((const __m512i*)(uv_buf + 32)); + y_buf += 32; + uv_buf += 64; + __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)); + __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)); + __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_storeu_si512((__m512i*)dst_ar30, dst0); + _mm512_storeu_si512((__m512i*)(dst_ar30 + 64), dst1); + dst_ar30 += 128; + width -= 32; + } + if (width > 0) { + uint32_t m32 = (1u << width) - 1u; + __m512i zmm4 = _mm512_maskz_loadu_epi16((__mmask32)m32, y_buf); + __m512i uv0 = _mm512_maskz_loadu_epi32((__mmask16)m32, uv_buf); + __m512i uv1 = _mm512_maskz_loadu_epi32((__mmask16)(m32 >> 16), uv_buf + 32); + __m256i lo = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8)); + __m256i hi = _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8)); + __m512i zmm3 = _mm512_inserti64x4(_mm512_zextsi256_si512(lo), hi, 1); + YUVTORGB16_WIN_AVX512BW; + PACKAR30_WIN_AVX512BW(dst0, dst1); + _mm512_mask_storeu_epi32(dst_ar30, (__mmask16)m32, dst0); + _mm512_mask_storeu_epi32(dst_ar30 + 64, (__mmask16)(m32 >> 16), dst1); + } + _mm256_zeroupper(); +} +#endif // HAS_P410TOAR30ROW_AVX512BW + +#if defined(HAS_ARGBTORGB24ROW_AVX512BW) || defined(HAS_ARGBTORAWROW_AVX512BW) +LIBYUV_TARGET_AVX512BW +static inline void ARGBToRGBRow_AVX512BW(const uint8_t* src, + uint8_t* dst, + int width, + const uint8_t* shuffler) { + const __m512i zmm_shuf = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)shuffler)); + const __m512i zmm_p0 = + _mm512_loadu_si512((const __m512i*)kPermdRGB24_0_AVX512BW); + const __m512i zmm_p1 = + _mm512_loadu_si512((const __m512i*)kPermdRGB24_1_AVX512BW); + const __m512i zmm_p2 = + _mm512_loadu_si512((const __m512i*)kPermdRGB24_2_AVX512BW); + + while (width >= 64) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src + 64)); + __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src + 128)); + __m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src + 192)); + src += 256; + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + __m512i out0 = _mm512_permutex2var_epi32(zmm0, zmm_p0, zmm1); + __m512i out1 = _mm512_permutex2var_epi32(zmm1, zmm_p1, zmm2); + __m512i out2 = _mm512_permutex2var_epi32(zmm2, zmm_p2, zmm3); + + _mm512_storeu_si512((__m512i*)dst, out0); + _mm512_storeu_si512((__m512i*)(dst + 64), out1); + _mm512_storeu_si512((__m512i*)(dst + 128), out2); + dst += 192; + width -= 64; + } + + if (width > 0) { + uint64_t m_in = (1ULL << width) - 1ULL; + __m512i zmm0 = _mm512_maskz_loadu_epi32((__mmask16)m_in, src); + __m512i zmm1 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 16), src + 64); + __m512i zmm2 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 32), src + 128); + __m512i zmm3 = _mm512_maskz_loadu_epi32((__mmask16)(m_in >> 48), src + 192); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + __m512i out0 = _mm512_permutex2var_epi32(zmm0, zmm_p0, zmm1); + __m512i out1 = _mm512_permutex2var_epi32(zmm1, zmm_p1, zmm2); + __m512i out2 = _mm512_permutex2var_epi32(zmm2, zmm_p2, zmm3); + + int bytes = width * 3; + __mmask64 k_out0 = + (bytes >= 64) ? ~(__mmask64)0 : (__mmask64)((1ULL << bytes) - 1ULL); + __mmask64 k_out1 = + (bytes <= 64) + ? 0 + : ((bytes >= 128) ? ~(__mmask64)0 + : (__mmask64)((1ULL << (bytes - 64)) - 1ULL)); + __mmask64 k_out2 = + (bytes <= 128) ? 0 : (__mmask64)((1ULL << (bytes - 128)) - 1ULL); + + _mm512_mask_storeu_epi8(dst, k_out0, out0); + _mm512_mask_storeu_epi8(dst + 64, k_out1, out1); + _mm512_mask_storeu_epi8(dst + 128, k_out2, out2); + } + _mm256_zeroupper(); +} +#endif // defined(HAS_ARGBTORGB24ROW_AVX512BW) || + // defined(HAS_ARGBTORAWROW_AVX512BW) + +#ifdef HAS_ARGBTORGB24ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToRGB24Row_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRGB24[0]); +} +#endif // HAS_ARGBTORGB24ROW_AVX512BW + +#ifdef HAS_ARGBTORAWROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToRAWRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + static const uint8_t kShuffleMaskARGBToRAW_AVX512[16] = { + 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, + 128u}; + ARGBToRGBRow_AVX512BW(src, dst, width, kShuffleMaskARGBToRAW_AVX512); +} +#endif // HAS_ARGBTORAWROW_AVX512BW + +#ifdef HAS_CONVERT16TO8ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void Convert16To8Row_AVX512BW(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + const __m512i zmm_255 = _mm512_set1_epi16(255); + do { + __m512i y0 = _mm512_loadu_si512((const __m512i*)src_y); + __m512i y1 = _mm512_loadu_si512((const __m512i*)(src_y + 32)); + y0 = _mm512_min_epu16(_mm512_mulhi_epu16(y0, zmm_scale), zmm_255); + y1 = _mm512_min_epu16(_mm512_mulhi_epu16(y1, zmm_scale), zmm_255); + _mm256_storeu_si256((__m256i*)dst_y, _mm512_cvtepi16_epi8(y0)); + _mm256_storeu_si256((__m256i*)(dst_y + 32), _mm512_cvtepi16_epi8(y1)); + src_y += 64; + dst_y += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_CONVERT16TO8ROW_AVX512BW + +#ifdef HAS_COPYROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void CopyRow_AVX512BW(const uint8_t* src, uint8_t* dst, int width) { + while (width >= 128) { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src); + __m512i z1 = _mm512_loadu_si512((const __m512i*)(src + 64)); + _mm512_storeu_si512((__m512i*)dst, z0); + _mm512_storeu_si512((__m512i*)(dst + 64), z1); + src += 128; + dst += 128; + width -= 128; + } + if (width >= 64) { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src); + _mm512_storeu_si512((__m512i*)dst, z0); + src += 64; + dst += 64; + width -= 64; + } + if (width > 0) { + __mmask64 k = (__mmask64)((1ULL << width) - 1ULL); + __m512i z0 = _mm512_maskz_loadu_epi8(k, src); + _mm512_mask_storeu_epi8(dst, k, z0); + } + _mm256_zeroupper(); +} +#endif // HAS_COPYROW_AVX512BW + +#ifdef HAS_HALFROW_16TO8_AVX512BW +LIBYUV_TARGET_AVX512BW +void HalfRow_16To8_AVX512BW(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + const __m512i zmm_255 = _mm512_set1_epi16(255); + do { + __m512i r0 = _mm512_loadu_si512((const __m512i*)src_uv); + __m512i r1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); + __m512i s0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride)); + __m512i s1 = + _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride + 32)); + r0 = _mm512_avg_epu16(r0, s0); + r1 = _mm512_avg_epu16(r1, s1); + r0 = _mm512_min_epu16(_mm512_mulhi_epu16(r0, zmm_scale), zmm_255); + r1 = _mm512_min_epu16(_mm512_mulhi_epu16(r1, zmm_scale), zmm_255); + _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(r0)); + _mm256_storeu_si256((__m256i*)(dst_uv + 32), _mm512_cvtepi16_epi8(r1)); + src_uv += 64; + dst_uv += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFROW_16TO8_AVX512BW + +#ifdef HAS_HALFWIDTHROW_16TO8_AVX512BW +LIBYUV_TARGET_AVX512BW +void HalfWidthRow_16To8_AVX512BW(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const __m512i zmm_mask16 = _mm512_set1_epi32(0x0000ffff); + const __m512i zmm_2 = _mm512_set1_epi32(2); + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + const __m512i zmm_255 = _mm512_set1_epi16(255); + do { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_uv); + __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_uv + 32)); + __m512i t0 = _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride)); + __m512i t1 = + _mm512_loadu_si512((const __m512i*)(src_uv + src_uv_stride + 32)); + __m512i sum0 = _mm512_add_epi32(_mm512_and_si512(s0, zmm_mask16), + _mm512_srli_epi32(s0, 16)); + sum0 = _mm512_add_epi32(sum0, _mm512_and_si512(t0, zmm_mask16)); + sum0 = _mm512_add_epi32(sum0, _mm512_srli_epi32(t0, 16)); + sum0 = _mm512_srli_epi32(_mm512_add_epi32(sum0, zmm_2), 2); + + __m512i sum1 = _mm512_add_epi32(_mm512_and_si512(s1, zmm_mask16), + _mm512_srli_epi32(s1, 16)); + sum1 = _mm512_add_epi32(sum1, _mm512_and_si512(t1, zmm_mask16)); + sum1 = _mm512_add_epi32(sum1, _mm512_srli_epi32(t1, 16)); + sum1 = _mm512_srli_epi32(_mm512_add_epi32(sum1, zmm_2), 2); + + __m512i avg16 = + _mm512_inserti64x4(_mm512_zextsi256_si512(_mm512_cvtepi32_epi16(sum0)), + _mm512_cvtepi32_epi16(sum1), 1); + avg16 = _mm512_min_epu16(_mm512_mulhi_epu16(avg16, zmm_scale), zmm_255); + _mm256_storeu_si256((__m256i*)dst_uv, _mm512_cvtepi16_epi8(avg16)); + src_uv += 64; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFWIDTHROW_16TO8_AVX512BW + +#ifdef HAS_SPLITUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void SplitUVRow_AVX512BW(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + do { + __m512i uv0 = _mm512_loadu_si512((const __m512i*)src_uv); + __m512i uv1 = _mm512_loadu_si512((const __m512i*)(src_uv + 64)); + _mm256_storeu_si256((__m256i*)dst_u, _mm512_cvtepi16_epi8(uv0)); + _mm256_storeu_si256((__m256i*)(dst_u + 32), _mm512_cvtepi16_epi8(uv1)); + _mm256_storeu_si256((__m256i*)dst_v, + _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv0, 8))); + _mm256_storeu_si256((__m256i*)(dst_v + 32), + _mm512_cvtepi16_epi8(_mm512_srli_epi16(uv1, 8))); + src_uv += 128; + dst_u += 64; + dst_v += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITUVROW_AVX512BW + +#ifdef HAS_CONVERT8TO16ROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void Convert8To16Row_AVX512BW(const uint8_t* src_y, + uint16_t* dst_y, + int bits, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(16 - bits); + do { + __m256i y0_8 = _mm256_loadu_si256((const __m256i*)src_y); + __m256i y1_8 = _mm256_loadu_si256((const __m256i*)(src_y + 32)); + __m512i y0 = _mm512_cvtepu8_epi16(y0_8); + __m512i y1 = _mm512_cvtepu8_epi16(y1_8); + y0 = _mm512_srl_epi16(_mm512_or_si512(y0, _mm512_slli_epi16(y0, 8)), + shift128); + y1 = _mm512_srl_epi16(_mm512_or_si512(y1, _mm512_slli_epi16(y1, 8)), + shift128); + _mm512_storeu_si512((__m512i*)dst_y, y0); + _mm512_storeu_si512((__m512i*)(dst_y + 32), y1); + src_y += 64; + dst_y += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_CONVERT8TO16ROW_AVX512BW + +#ifdef HAS_J400TOARGBROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void J400ToARGBRow_AVX512BW(const uint8_t* src_y, + uint8_t* dst_argb, + int width) { + static const uint8_t kShuffleMaskJ400ToARGB_AVX512[64] = { + 0u, 0u, 0u, 128u, 1u, 1u, 1u, 128u, 2u, 2u, 2u, 128u, 3u, + 3u, 3u, 128u, 4u, 4u, 4u, 128u, 5u, 5u, 5u, 128u, 6u, 6u, + 6u, 128u, 7u, 7u, 7u, 128u, 8u, 8u, 8u, 128u, 9u, 9u, 9u, + 128u, 10u, 10u, 10u, 128u, 11u, 11u, 11u, 128u, 12u, 12u, 12u, 128u, + 13u, 13u, 13u, 128u, 14u, 14u, 14u, 128u, 15u, 15u, 15u, 128u}; + const __m512i zmm_mask = + _mm512_loadu_si512((const __m512i*)kShuffleMaskJ400ToARGB_AVX512); + const __m512i zmm_alpha = _mm512_set1_epi32((int)0xff000000u); + do { + __m512i z0 = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)src_y)); + __m512i z1 = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)(src_y + 16))); + z0 = _mm512_or_si512(_mm512_shuffle_epi8(z0, zmm_mask), zmm_alpha); + z1 = _mm512_or_si512(_mm512_shuffle_epi8(z1, zmm_mask), zmm_alpha); + _mm512_storeu_si512((__m512i*)dst_argb, z0); + _mm512_storeu_si512((__m512i*)(dst_argb + 64), z1); + src_y += 32; + dst_argb += 128; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_J400TOARGBROW_AVX512BW + +#ifdef HAS_MERGEUVROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void MergeUVRow_AVX512BW(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width) { + do { + __m512i u16 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_u)); + __m512i v16 = _mm512_cvtepu8_epi16(_mm256_loadu_si256((const __m256i*)src_v)); + __m512i uv = _mm512_or_si512(u16, _mm512_slli_epi16(v16, 8)); + _mm512_storeu_si512((__m512i*)dst_uv, uv); + src_u += 32; + src_v += 32; + dst_uv += 64; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEUVROW_AVX512BW + +#ifdef HAS_MULTIPLYROW_16_AVX512BW +LIBYUV_TARGET_AVX512BW +void MultiplyRow_16_AVX512BW(const uint16_t* src_y, + uint16_t* dst_y, + int scale, + int width) { + const __m512i zmm_scale = _mm512_set1_epi16((short)scale); + do { + __m512i z0 = _mm512_loadu_si512((const __m512i*)src_y); + __m512i z1 = _mm512_loadu_si512((const __m512i*)(src_y + 32)); + z0 = _mm512_mullo_epi16(z0, zmm_scale); + z1 = _mm512_mullo_epi16(z1, zmm_scale); + _mm512_storeu_si512((__m512i*)dst_y, z0); + _mm512_storeu_si512((__m512i*)(dst_y + 32), z1); + src_y += 64; + dst_y += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MULTIPLYROW_16_AVX512BW + +#ifdef HAS_ARGBTOYMATRIXROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToYMatrixRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_y, + int width, + const struct ArgbConstants* c) { + const __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + const __m512i zmm_80 = _mm512_set1_epi8((char)0x80); + const __m512i zmm_one = _mm512_set1_epi16(1); + const __m512i zmm_coeff = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToY)); + __m512i zmm_add = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kAddY)); + + __m512i zmm_bias = _mm512_maddubs_epi16(zmm_coeff, zmm_80); + zmm_bias = _mm512_madd_epi16(zmm_bias, zmm_one); + zmm_bias = _mm512_packs_epi32(zmm_bias, zmm_bias); + zmm_add = _mm512_sub_epi16(zmm_add, zmm_bias); + + while (width > 0) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); + __m512i zmm3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192)); + + zmm0 = _mm512_xor_si512(zmm0, zmm_80); + zmm1 = _mm512_xor_si512(zmm1, zmm_80); + zmm2 = _mm512_xor_si512(zmm2, zmm_80); + zmm3 = _mm512_xor_si512(zmm3, zmm_80); + + zmm0 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm0), zmm_one); + zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm1), zmm_one); + zmm0 = _mm512_packs_epi32(zmm0, zmm1); + + zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm2), zmm_one); + zmm3 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm_coeff, zmm3), zmm_one); + zmm2 = _mm512_packs_epi32(zmm2, zmm3); + + zmm0 = _mm512_srli_epi16(_mm512_add_epi16(zmm0, zmm_add), 8); + zmm2 = _mm512_srli_epi16(_mm512_add_epi16(zmm2, zmm_add), 8); + + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm2)); + _mm512_storeu_si512((__m512i*)dst_y, zmm0); + + src_argb += 256; + dst_y += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOYMATRIXROW_AVX512BW + +#ifdef HAS_ARGBTOUV444MATRIXROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToUV444MatrixRow_AVX512BW(const uint8_t* src_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + const __m512i zmm_u = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + const __m512i zmm_v = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + const __m512i zmm_one = _mm512_set1_epi16(1); + const __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + const __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + + while (width > 0) { + __m512i s0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i s1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i s2 = _mm512_loadu_si512((const __m512i*)(src_argb + 128)); + __m512i s3 = _mm512_loadu_si512((const __m512i*)(src_argb + 192)); + + __m512i u0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_u), zmm_one); + __m512i u1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_u), zmm_one); + __m512i u2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_u), zmm_one); + __m512i u3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_u), zmm_one); + __m512i u01 = + _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u0, u1)), + 8); + __m512i u23 = + _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(u2, u3)), + 8); + _mm512_storeu_si512( + (__m512i*)dst_u, + _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(u01, u23))); + + __m512i v0 = _mm512_madd_epi16(_mm512_maddubs_epi16(s0, zmm_v), zmm_one); + __m512i v1 = _mm512_madd_epi16(_mm512_maddubs_epi16(s1, zmm_v), zmm_one); + __m512i v2 = _mm512_madd_epi16(_mm512_maddubs_epi16(s2, zmm_v), zmm_one); + __m512i v3 = _mm512_madd_epi16(_mm512_maddubs_epi16(s3, zmm_v), zmm_one); + __m512i v01 = + _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v0, v1)), + 8); + __m512i v23 = + _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, _mm512_packs_epi32(v2, v3)), + 8); + _mm512_storeu_si512( + (__m512i*)dst_v, + _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(v01, v23))); + + src_argb += 256; + dst_u += 64; + dst_v += 64; + width -= 64; + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOUV444MATRIXROW_AVX512BW + +#ifdef HAS_ARGBTOUVMATRIXROW_AVX512BW +LIBYUV_TARGET_AVX512BW +void ARGBToUVMatrixRow_AVX512BW(const uint8_t* src_argb, + int src_stride_argb, + uint8_t* dst_u, + uint8_t* dst_v, + int width, + const struct ArgbConstants* c) { + static const uint32_t kPermdARGBToUV_AVX512BW[16] = { + 0, 1, 4, 5, 8, 9, 12, 13, 2, 3, 6, 7, 10, 11, 14, 15}; + static const uint8_t kShuffleAARRGGBB_AVX512BW[16] = { + 0, 4, 1, 5, 2, 6, 3, 7, 8, 12, 9, 13, 10, 14, 11, 15}; + const __m512i zmm_u = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToU)); + const __m512i zmm_v = + _mm512_broadcast_i32x4(_mm_loadu_si128((const __m128i*)c->kRGBToV)); + const __m512i zmm_0101 = _mm512_set1_epi16(0x0101); + const __m512i zmm_one = _mm512_set1_epi16(1); + const __m512i zmm_8000 = _mm512_set1_epi16((short)0x8000); + const __m512i zmm_zero = _mm512_setzero_si512(); + const __m512i zmm_shuf = _mm512_broadcast_i32x4( + _mm_loadu_si128((const __m128i*)kShuffleAARRGGBB_AVX512BW)); + const __m512i zmm_perm_y = + _mm512_loadu_si512((const __m512i*)kPermdARGBToY_AVX512BW); + const __m512i zmm_perm_uv = + _mm512_loadu_si512((const __m512i*)kPermdARGBToUV_AVX512BW); + + while (width >= 32) { + __m512i zmm0 = _mm512_loadu_si512((const __m512i*)src_argb); + __m512i zmm1 = _mm512_loadu_si512((const __m512i*)(src_argb + 64)); + __m512i zmm2 = + _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb)); + __m512i zmm3 = + _mm512_loadu_si512((const __m512i*)(src_argb + src_stride_argb + 64)); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101); + zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101); + zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101); + zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101); + + zmm0 = _mm512_add_epi16(zmm0, zmm2); + zmm1 = _mm512_add_epi16(zmm1, zmm3); + + zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero); + zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero); + + zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); + + zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one); + zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one); + zmm0 = _mm512_packs_epi32(zmm1, zmm2); + zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); + + _mm_storeu_si128((__m128i*)dst_u, _mm512_castsi512_si128(zmm0)); + _mm_storeu_si128((__m128i*)dst_v, _mm512_extracti32x4_epi32(zmm0, 1)); + + src_argb += 128; + dst_u += 16; + dst_v += 16; + width -= 32; + } + if (width > 0) { + uint32_t mask = (1u << width) - 1u; + __mmask16 k1 = (__mmask16)mask; + __mmask16 k2 = (__mmask16)(mask >> 16); + __mmask16 k3 = ~k1; + __mmask16 k4 = ~k2; + __m512i zmm0 = _mm512_maskz_loadu_epi32(k1, src_argb); + __m512i zmm1 = _mm512_maskz_loadu_epi32(k2, src_argb + 64); + __m512i zmm2 = _mm512_maskz_loadu_epi32(k1, src_argb + src_stride_argb); + __m512i zmm3 = + _mm512_maskz_loadu_epi32(k2, src_argb + src_stride_argb + 64); + + zmm0 = _mm512_mask_shuffle_epi32(zmm0, k3, zmm0, (_MM_PERM_ENUM)0xa0); + zmm1 = _mm512_mask_shuffle_epi32(zmm1, k4, zmm1, (_MM_PERM_ENUM)0xa0); + zmm2 = _mm512_mask_shuffle_epi32(zmm2, k3, zmm2, (_MM_PERM_ENUM)0xa0); + zmm3 = _mm512_mask_shuffle_epi32(zmm3, k4, zmm3, (_MM_PERM_ENUM)0xa0); + + zmm0 = _mm512_shuffle_epi8(zmm0, zmm_shuf); + zmm1 = _mm512_shuffle_epi8(zmm1, zmm_shuf); + zmm2 = _mm512_shuffle_epi8(zmm2, zmm_shuf); + zmm3 = _mm512_shuffle_epi8(zmm3, zmm_shuf); + + zmm0 = _mm512_maddubs_epi16(zmm0, zmm_0101); + zmm1 = _mm512_maddubs_epi16(zmm1, zmm_0101); + zmm2 = _mm512_maddubs_epi16(zmm2, zmm_0101); + zmm3 = _mm512_maddubs_epi16(zmm3, zmm_0101); + + zmm0 = _mm512_add_epi16(zmm0, zmm2); + zmm1 = _mm512_add_epi16(zmm1, zmm3); + + zmm0 = _mm512_avg_epu16(_mm512_srli_epi16(zmm0, 1), zmm_zero); + zmm1 = _mm512_avg_epu16(_mm512_srli_epi16(zmm1, 1), zmm_zero); + + zmm0 = _mm512_permutexvar_epi32(zmm_perm_uv, _mm512_packus_epi16(zmm0, zmm1)); + + zmm1 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_u), zmm_one); + zmm2 = _mm512_madd_epi16(_mm512_maddubs_epi16(zmm0, zmm_v), zmm_one); + zmm0 = _mm512_packs_epi32(zmm1, zmm2); + zmm0 = _mm512_srli_epi16(_mm512_sub_epi16(zmm_8000, zmm0), 8); + zmm0 = _mm512_permutexvar_epi32(zmm_perm_y, _mm512_packus_epi16(zmm0, zmm0)); + + int halfwidth = (width + 1) >> 1; + __mmask16 k_out = (__mmask16)((1u << halfwidth) - 1u); + _mm_mask_storeu_epi8(dst_u, k_out, _mm512_castsi512_si128(zmm0)); + _mm_mask_storeu_epi8(dst_v, k_out, _mm512_extracti32x4_epi32(zmm0, 1)); + } + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOUVMATRIXROW_AVX512BW + +#endif // defined(__x86_64__) || defined(_M_X64) + +#ifdef HAS_COPYROW_AVX +LIBYUV_TARGET_AVX +void CopyRow_AVX(const uint8_t* src, uint8_t* dst, int width) { + do { + __m256i y0 = _mm256_loadu_si256((const __m256i*)src); + __m256i y1 = _mm256_loadu_si256((const __m256i*)(src + 32)); + _mm256_storeu_si256((__m256i*)dst, y0); + _mm256_storeu_si256((__m256i*)(dst + 32), y1); + src += 64; + dst += 64; + width -= 64; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_COPYROW_AVX + +#ifdef HAS_DETILETOYUY2_AVX2 +LIBYUV_TARGET_AVX2 +void DetileToYUY2_AVX2(const uint8_t* src_y, + ptrdiff_t src_y_tile_stride, + const uint8_t* src_uv, + ptrdiff_t src_uv_tile_stride, + uint8_t* dst_yuy2, + int width) { + do { + __m128i y0 = _mm_loadu_si128((const __m128i*)src_y); + __m128i y1 = _mm_loadu_si128((const __m128i*)(src_y + src_y_tile_stride)); + __m128i uv0 = _mm_loadu_si128((const __m128i*)src_uv); + __m128i uv1 = + _mm_loadu_si128((const __m128i*)(src_uv + src_uv_tile_stride)); + __m256i y = _mm256_inserti128_si256(_mm256_castsi128_si256(y0), y1, 1); + __m256i uv = _mm256_inserti128_si256(_mm256_castsi128_si256(uv0), uv1, 1); + __m256i lo = _mm256_unpacklo_epi8(y, uv); + __m256i hi = _mm256_unpackhi_epi8(y, uv); + _mm256_storeu_si256((__m256i*)dst_yuy2, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_yuy2 + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_y += 2 * src_y_tile_stride; + src_uv += 2 * src_uv_tile_stride; + dst_yuy2 += 64; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_DETILETOYUY2_AVX2 + +#ifdef HAS_UNPACKMT2T_AVX2 +LIBYUV_TARGET_AVX2 +void UnpackMT2T_AVX2(const uint8_t* src, uint16_t* dst, size_t size) { + const __m256i mask_00c0 = _mm256_set1_epi16(0x00c0); + for (size_t i = 0; i < size; i += 80) { + __m256i low16 = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)src)); + __m256i l0 = _mm256_and_si256(_mm256_slli_epi16(low16, 6), mask_00c0); + __m256i l1 = _mm256_and_si256(_mm256_slli_epi16(low16, 4), mask_00c0); + __m256i l2 = _mm256_and_si256(_mm256_slli_epi16(low16, 2), mask_00c0); + __m256i l3 = _mm256_and_si256(low16, mask_00c0); + + __m256i u0 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 16))); + __m256i u1 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 32))); + __m256i u2 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 48))); + __m256i u3 = + _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)(src + 64))); + + u0 = _mm256_or_si256( + _mm256_or_si256(_mm256_slli_epi16(u0, 8), _mm256_srli_epi16(u0, 2)), + l0); + u1 = _mm256_or_si256( + _mm256_or_si256(_mm256_slli_epi16(u1, 8), _mm256_srli_epi16(u1, 2)), + l1); + u2 = _mm256_or_si256( + _mm256_or_si256(_mm256_slli_epi16(u2, 8), _mm256_srli_epi16(u2, 2)), + l2); + u3 = _mm256_or_si256( + _mm256_or_si256(_mm256_slli_epi16(u3, 8), _mm256_srli_epi16(u3, 2)), + l3); + + _mm256_storeu_si256((__m256i*)dst, u0); + _mm256_storeu_si256((__m256i*)(dst + 16), u1); + _mm256_storeu_si256((__m256i*)(dst + 32), u2); + _mm256_storeu_si256((__m256i*)(dst + 48), u3); + + src += 80; + dst += 64; + } + _mm256_zeroupper(); +} +#endif // HAS_UNPACKMT2T_AVX2 + +#ifdef HAS_ARGBGRAYROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBGrayRow_AVX2(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + static const uint8_t kShufGrayDword[16] = {0, 0, 0, 0x80, 4, 4, 4, 0x80, + 8, 8, 8, 0x80, 12, 12, 12, 0x80}; + const __m256i c_coeff = _mm256_set1_epi32(0x004d961d); + const __m256i c_sub128 = _mm256_set1_epi8((char)0x80); + const __m256i c_one16 = _mm256_set1_epi16(1); + const __m256i c_bias = _mm256_set1_epi32(32896); + const __m256i c_shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufGrayDword)); + const __m256i c_alpha = _mm256_set1_epi32((int)0xff000000u); + + while (width >= 8) { + __m256i s = _mm256_loadu_si256((const __m256i*)src_argb); + src_argb += 32; + __m256i y16 = _mm256_maddubs_epi16(c_coeff, _mm256_sub_epi8(s, c_sub128)); + __m256i y32 = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_madd_epi16(y16, c_one16), c_bias), 8); + __m256i out = _mm256_or_si256(_mm256_and_si256(s, c_alpha), + _mm256_shuffle_epi8(y32, c_shuf)); + _mm256_storeu_si256((__m256i*)dst_argb, out); + dst_argb += 32; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + ARGBGrayRow_C(src_argb, dst_argb, width); + } +} +#endif // HAS_ARGBGRAYROW_AVX2 + +#ifdef HAS_MERGERGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeRGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_rgb, + int width) { + static const uint8_t kMergeRGBShuf[5][16] = { + {0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u, 10u, 5u}, + {5u, 0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u, 10u}, + {10u, 5u, 0u, 11u, 6u, 1u, 12u, 7u, 2u, 13u, 8u, 3u, 14u, 9u, 4u, 15u}, + {0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, + 0u}, + {0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, + 0u}}; + const __m256i shuf_r = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[0])); + const __m256i shuf_g = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[1])); + const __m256i shuf_b = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[2])); + const __m256i m1 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[3])); + const __m256i m2 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kMergeRGBShuf[4])); + + do { + __m256i r = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_r), shuf_r); + __m256i g = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_g), shuf_g); + __m256i b = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_b), shuf_b); + + __m256i c0 = _mm256_blendv_epi8(_mm256_blendv_epi8(r, g, m1), b, m2); + __m256i c1 = _mm256_blendv_epi8(_mm256_blendv_epi8(g, b, m1), r, m2); + __m256i c2 = _mm256_blendv_epi8(_mm256_blendv_epi8(b, r, m1), g, m2); + + _mm256_storeu_si256((__m256i*)dst_rgb, + _mm256_permute2x128_si256(c0, c1, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_rgb + 32), + _mm256_permute2x128_si256(c2, c0, 0x30)); + _mm256_storeu_si256((__m256i*)(dst_rgb + 64), + _mm256_permute2x128_si256(c1, c2, 0x31)); + + src_r += 32; + src_g += 32; + src_b += 32; + dst_rgb += 96; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGERGBROW_AVX2 + +#ifdef HAS_RAWTORGBAROW_AVX2 +LIBYUV_TARGET_AVX2 +void RAWToRGBARow_AVX2(const uint8_t* src_raw, uint8_t* dst_rgba, int width) { + static const uint8_t kShufRAWToRGBA_Lo[16] = {0x80, 2u, 1u, 0u, 0x80, 5u, + 4u, 3u, 0x80, 8u, 7u, 6u, + 0x80, 11u, 10u, 9u}; + static const uint8_t kShufRAWToRGBA_Hi[16] = {0x80, 6u, 5u, 4u, 0x80, 9u, + 8u, 7u, 0x80, 12u, 11u, 10u, + 0x80, 15u, 14u, 13u}; + const __m256i shuf_lo = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufRAWToRGBA_Lo)); + const __m256i shuf_hi = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufRAWToRGBA_Hi)); + const __m256i alpha = _mm256_set1_epi32(0x000000ff); + + do { + __m128i s0 = _mm_loadu_si128((const __m128i*)src_raw); + __m128i s1 = _mm_loadu_si128((const __m128i*)(src_raw + 12)); + __m128i s2 = _mm_loadu_si128((const __m128i*)(src_raw + 20)); + __m128i s3 = _mm_loadu_si128((const __m128i*)(src_raw + 32)); + __m256i y0 = _mm256_inserti128_si256(_mm256_castsi128_si256(s0), s1, 1); + __m256i y1 = _mm256_inserti128_si256(_mm256_castsi128_si256(s2), s3, 1); + y0 = _mm256_or_si256(_mm256_shuffle_epi8(y0, shuf_lo), alpha); + y1 = _mm256_or_si256(_mm256_shuffle_epi8(y1, shuf_hi), alpha); + _mm256_storeu_si256((__m256i*)dst_rgba, y0); + _mm256_storeu_si256((__m256i*)(dst_rgba + 32), y1); + src_raw += 48; + dst_rgba += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_RAWTORGBAROW_AVX2 + +#ifdef HAS_HALFROW_AVX2 +LIBYUV_TARGET_AVX2 +void HalfRow_AVX2(const uint8_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int width) { + do { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride)); + _mm256_storeu_si256((__m256i*)dst_uv, _mm256_avg_epu8(r0, r1)); + src_uv += 32; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFROW_AVX2 + +#ifdef HAS_HALFROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void HalfRow_16_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint16_t* dst_uv, + int width) { + do { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride)); + _mm256_storeu_si256((__m256i*)dst_uv, _mm256_avg_epu16(r0, r1)); + src_uv += 16; + dst_uv += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFROW_16_AVX2 + +#ifdef HAS_CONVERT16TO8ROW_AVX2 +LIBYUV_TARGET_AVX2 +void Convert16To8Row_AVX2(const uint16_t* src_y, + uint8_t* dst_y, + int scale, + int width) { + const __m256i ymm_scale = _mm256_set1_epi16((short)scale); + const __m256i ymm_255 = _mm256_set1_epi16(255); + do { + __m256i y0 = _mm256_loadu_si256((const __m256i*)src_y); + __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_y + 16)); + y0 = _mm256_min_epu16(_mm256_mulhi_epu16(y0, ymm_scale), ymm_255); + y1 = _mm256_min_epu16(_mm256_mulhi_epu16(y1, ymm_scale), ymm_255); + __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(y0, y1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_y, out); + src_y += 32; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_CONVERT16TO8ROW_AVX2 + +#ifdef HAS_HALFROW_16TO8_AVX2 +LIBYUV_TARGET_AVX2 +void HalfRow_16To8_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const __m256i ymm_scale = _mm256_set1_epi16((short)scale); + const __m256i ymm_255 = _mm256_set1_epi16(255); + do { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16)); + __m256i s0 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride)); + __m256i s1 = + _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride + 16)); + r0 = _mm256_min_epu16(_mm256_mulhi_epu16(_mm256_avg_epu16(r0, s0), ymm_scale), + ymm_255); + r1 = _mm256_min_epu16(_mm256_mulhi_epu16(_mm256_avg_epu16(r1, s1), ymm_scale), + ymm_255); + __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(r0, r1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, out); + src_uv += 32; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFROW_16TO8_AVX2 + +#ifdef HAS_HALFWIDTHROW_16TO8_AVX2 +LIBYUV_TARGET_AVX2 +void HalfWidthRow_16To8_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + const __m256i ymm_mask16 = _mm256_set1_epi32(0x0000ffff); + const __m256i ymm_2 = _mm256_set1_epi32(2); + const __m256i ymm_scale = _mm256_set1_epi16((short)scale); + const __m256i ymm_255 = _mm256_set1_epi16(255); + do { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i s1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16)); + __m256i t0 = _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride)); + __m256i t1 = + _mm256_loadu_si256((const __m256i*)(src_uv + src_uv_stride + 16)); + + __m256i sum0 = _mm256_add_epi32(_mm256_and_si256(s0, ymm_mask16), + _mm256_srli_epi32(s0, 16)); + sum0 = _mm256_add_epi32(sum0, _mm256_and_si256(t0, ymm_mask16)); + sum0 = _mm256_add_epi32(sum0, _mm256_srli_epi32(t0, 16)); + sum0 = _mm256_srli_epi32(_mm256_add_epi32(sum0, ymm_2), 2); + + __m256i sum1 = _mm256_add_epi32(_mm256_and_si256(s1, ymm_mask16), + _mm256_srli_epi32(s1, 16)); + sum1 = _mm256_add_epi32(sum1, _mm256_and_si256(t1, ymm_mask16)); + sum1 = _mm256_add_epi32(sum1, _mm256_srli_epi32(t1, 16)); + sum1 = _mm256_srli_epi32(_mm256_add_epi32(sum1, ymm_2), 2); + + __m256i avg16 = + _mm256_permute4x64_epi64(_mm256_packus_epi32(sum0, sum1), 0xd8); + avg16 = _mm256_min_epu16(_mm256_mulhi_epu16(avg16, ymm_scale), ymm_255); + __m256i out = + _mm256_permute4x64_epi64(_mm256_packus_epi16(avg16, avg16), 0xd8); + _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(out)); + src_uv += 32; + dst_uv += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_HALFWIDTHROW_16TO8_AVX2 + +#ifdef HAS_HALFWIDTHROW_16TO8_ODD_AVX2 +LIBYUV_TARGET_AVX2 +void HalfWidthRow_16To8_Odd_AVX2(const uint16_t* src_uv, + ptrdiff_t src_uv_stride, + uint8_t* dst_uv, + int scale, + int width) { + int n = (width - 1) & ~15; + if (n > 0) { + HalfWidthRow_16To8_AVX2(src_uv, src_uv_stride, dst_uv, scale, n); + src_uv += n * 2; + dst_uv += n; + width -= n; + } + HalfWidthRow_16To8_Odd_C(src_uv, src_uv_stride, dst_uv, scale, width); +} +#endif // HAS_HALFWIDTHROW_16TO8_ODD_AVX2 + +#ifdef HAS_GAUSSCOL_AVX2 +LIBYUV_TARGET_AVX2 +void GaussCol_AVX2(const uint16_t* src0, + const uint16_t* src1, + const uint16_t* src2, + const uint16_t* src3, + const uint16_t* src4, + uint32_t* dst, + int width) { + while (width >= 8) { + __m256i v0 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src0)); + __m256i v1 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src1)); + __m256i v2 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src2)); + __m256i v3 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src3)); + __m256i v4 = _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src4)); + __m256i v04 = _mm256_add_epi32(v0, v4); + __m256i v13 = _mm256_slli_epi32(_mm256_add_epi32(v1, v3), 2); + __m256i v2_6 = + _mm256_add_epi32(_mm256_slli_epi32(v2, 2), _mm256_slli_epi32(v2, 1)); + _mm256_storeu_si256((__m256i*)dst, + _mm256_add_epi32(_mm256_add_epi32(v04, v13), v2_6)); + src0 += 8; + src1 += 8; + src2 += 8; + src3 += 8; + src4 += 8; + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + GaussCol_C(src0, src1, src2, src3, src4, dst, width); + } +} +#endif // HAS_GAUSSCOL_AVX2 + +#ifdef HAS_GAUSSCOL_F32_AVX2 +LIBYUV_TARGET_AVX2 +void GaussCol_F32_AVX2(const float* src0, + const float* src1, + const float* src2, + const float* src3, + const float* src4, + float* dst, + int width) { + const __m256 c4 = _mm256_set1_ps(4.0f); + const __m256 c6 = _mm256_set1_ps(6.0f); + while (width >= 8) { + __m256 s0 = _mm256_loadu_ps(src0); + __m256 s1 = _mm256_loadu_ps(src1); + __m256 s2 = _mm256_loadu_ps(src2); + __m256 s3 = _mm256_loadu_ps(src3); + __m256 s4 = _mm256_loadu_ps(src4); + __m256 sum = _mm256_add_ps(s0, _mm256_mul_ps(s1, c4)); + sum = _mm256_add_ps(sum, _mm256_mul_ps(s2, c6)); + sum = _mm256_add_ps(sum, _mm256_mul_ps(s3, c4)); + sum = _mm256_add_ps(sum, s4); + _mm256_storeu_ps(dst, sum); + src0 += 8; + src1 += 8; + src2 += 8; + src3 += 8; + src4 += 8; + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + GaussCol_F32_C(src0, src1, src2, src3, src4, dst, width); + } +} +#endif // HAS_GAUSSCOL_F32_AVX2 + +#ifdef HAS_SCALEMAXSAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +float ScaleMaxSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + __m256 vmax = _mm256_setzero_ps(); + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + vmax = _mm256_max_ps(v, vmax); + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + __m128 m128 = + _mm_max_ps(_mm256_extractf128_ps(vmax, 1), _mm256_castps256_ps128(vmax)); + m128 = _mm_max_ps(_mm_movehl_ps(m128, m128), m128); + m128 = _mm_max_ss(_mm_shuffle_ps(m128, m128, 1), m128); + float fmax = _mm_cvtss_f32(m128); + _mm256_zeroupper(); + for (; width > 0; --width) { + float v = *src++; + float vs = v * scale; + fmax = (v > fmax) ? v : fmax; + *dst++ = vs; + } + return fmax; +} +#endif // HAS_SCALEMAXSAMPLES_AVX2 + +#ifdef HAS_SCALESUMSAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +float ScaleSumSamples_AVX2(const float* src, + float* dst, + float scale, + int width) { + __m256 vsum = _mm256_setzero_ps(); + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + vsum = _mm256_add_ps(vsum, _mm256_mul_ps(v, v)); + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + __m128 s128 = + _mm_add_ps(_mm256_castps256_ps128(vsum), _mm256_extractf128_ps(vsum, 1)); + s128 = _mm_add_ps(s128, _mm_movehl_ps(s128, s128)); + s128 = _mm_add_ss(s128, _mm_shuffle_ps(s128, s128, 1)); + float fsum = _mm_cvtss_f32(s128); + _mm256_zeroupper(); + for (; width > 0; --width) { + float v = *src++; + fsum += v * v; + *dst++ = v * scale; + } + return fsum; +} +#endif // HAS_SCALESUMSAMPLES_AVX2 + +#ifdef HAS_SCALESAMPLES_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleSamples_AVX2(const float* src, float* dst, float scale, int width) { + const __m256 vscale = _mm256_set1_ps(scale); + while (width >= 8) { + __m256 v = _mm256_loadu_ps(src); + src += 8; + _mm256_storeu_ps(dst, _mm256_mul_ps(v, vscale)); + dst += 8; + width -= 8; + } + _mm256_zeroupper(); + if (width > 0) { + ScaleSamples_C(src, dst, scale, width); + } +} +#endif // HAS_SCALESAMPLES_AVX2 + +#if defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2) +LIBYUV_TARGET_AVX2 +static inline void ARGBToRGBRow_AVX2(const uint8_t* src, + uint8_t* dst, + int width, + const uint8_t* shuffler) { + static const uint32_t kPermdRGB24_AVX[8] = {0, 1, 2, 4, 5, 6, 3, 7}; + const __m256i ymm_shuf = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)shuffler)); + const __m256i ymm_perm = + _mm256_loadu_si256((const __m256i*)kPermdRGB24_AVX); + do { + __m256i ymm0 = _mm256_loadu_si256((const __m256i*)src); + __m256i ymm1 = _mm256_loadu_si256((const __m256i*)(src + 32)); + __m256i ymm2 = _mm256_loadu_si256((const __m256i*)(src + 64)); + __m256i ymm3 = _mm256_loadu_si256((const __m256i*)(src + 96)); + src += 128; + + ymm0 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm0, ymm_shuf), + ymm_perm); + ymm1 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm1, ymm_shuf), + ymm_perm); + ymm2 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm2, ymm_shuf), + ymm_perm); + ymm3 = _mm256_permutevar8x32_epi32(_mm256_shuffle_epi8(ymm3, ymm_shuf), + ymm_perm); + + _mm256_storeu_si256( + (__m256i*)dst, + _mm256_or_si256(ymm0, _mm256_permute4x64_epi64(ymm1, 0x3f))); + _mm256_storeu_si256( + (__m256i*)(dst + 32), + _mm256_or_si256(_mm256_permute4x64_epi64(ymm1, 0xf9), + _mm256_permute4x64_epi64(ymm2, 0x4f))); + _mm256_storeu_si256( + (__m256i*)(dst + 64), + _mm256_or_si256(_mm256_permute4x64_epi64(ymm2, 0xfe), + _mm256_permute4x64_epi64(ymm3, 0x93))); + dst += 96; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // defined(HAS_ARGBTORGB24ROW_AVX2) || defined(HAS_ARGBTORAWROW_AVX2) + +#ifdef HAS_ARGBTORGB24ROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBToRGB24Row_AVX2(const uint8_t* src, uint8_t* dst, int width) { + ARGBToRGBRow_AVX2(src, dst, width, kShuffleMaskARGBToRGB24[0]); +} +#endif // HAS_ARGBTORGB24ROW_AVX2 + +#ifdef HAS_ARGBTORAWROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBToRAWRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { + static const uint8_t kShuffleMaskARGBToRAW_AVX2[16] = { + 2u, 1u, 0u, 6u, 5u, 4u, 10u, 9u, 8u, 14u, 13u, 12u, 128u, 128u, 128u, + 128u}; + ARGBToRGBRow_AVX2(src, dst, width, kShuffleMaskARGBToRAW_AVX2); +} +#endif // HAS_ARGBTORAWROW_AVX2 + +#ifdef HAS_SPLITUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void SplitUVRow_AVX2(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i uv0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i uv1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32)); + __m256i u = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_and_si256(uv0, mask_00ff), + _mm256_and_si256(uv1, mask_00ff)), + 0xd8); + __m256i v = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_srli_epi16(uv0, 8), + _mm256_srli_epi16(uv1, 8)), + 0xd8); + _mm256_storeu_si256((__m256i*)dst_u, u); + _mm256_storeu_si256((__m256i*)dst_v, v); + src_uv += 64; + dst_u += 32; + dst_v += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITUVROW_AVX2 + +#ifdef HAS_AR64TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void AR64ToARGBRow_AVX2(const uint16_t* src_ar64, + uint8_t* dst_argb, + int width) { + do { + __m256i s0 = _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_ar64), + 8); + __m256i s1 = _mm256_srli_epi16( + _mm256_loadu_si256((const __m256i*)(src_ar64 + 16)), 8); + __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, out); + src_ar64 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_AR64TOARGBROW_AVX2 + +#ifdef HAS_AB64TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void AB64ToARGBRow_AVX2(const uint16_t* src_ab64, + uint8_t* dst_argb, + int width) { + static const uint8_t kShufSwapRB[16] = {2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u, + 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u}; + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufSwapRB)); + do { + __m256i s0 = _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_ab64), + 8); + __m256i s1 = _mm256_srli_epi16( + _mm256_loadu_si256((const __m256i*)(src_ab64 + 16)), 8); + __m256i out = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + out = _mm256_shuffle_epi8(out, shuf); + _mm256_storeu_si256((__m256i*)dst_argb, out); + src_ab64 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_AB64TOARGBROW_AVX2 + +#ifdef HAS_ARGBTOAR64ROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBToAR64Row_AVX2(const uint8_t* src_argb, + uint16_t* dst_ar64, + int width) { + do { + __m256i s = _mm256_permute4x64_epi64( + _mm256_loadu_si256((const __m256i*)src_argb), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ar64, _mm256_unpacklo_epi8(s, s)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 16), _mm256_unpackhi_epi8(s, s)); + src_argb += 32; + dst_ar64 += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOAR64ROW_AVX2 + +#ifdef HAS_ARGBTOAB64ROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBToAB64Row_AVX2(const uint8_t* src_argb, + uint16_t* dst_ab64, + int width) { + static const uint8_t kShufSwapRB[16] = {2u, 1u, 0u, 3u, 6u, 5u, 4u, 7u, + 10u, 9u, 8u, 11u, 14u, 13u, 12u, 15u}; + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufSwapRB)); + do { + __m256i s = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)src_argb), shuf); + s = _mm256_permute4x64_epi64(s, 0xd8); + _mm256_storeu_si256((__m256i*)dst_ab64, _mm256_unpacklo_epi8(s, s)); + _mm256_storeu_si256((__m256i*)(dst_ab64 + 16), _mm256_unpackhi_epi8(s, s)); + src_argb += 32; + dst_ab64 += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBTOAB64ROW_AVX2 + +#ifdef HAS_ARGBADDROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBAddRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + do { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_argb); + __m256i s1 = _mm256_loadu_si256((const __m256i*)src_argb1); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_adds_epu8(s0, s1)); + src_argb += 32; + src_argb1 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBADDROW_AVX2 + +#ifdef HAS_ARGBSUBTRACTROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBSubtractRow_AVX2(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + do { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_argb); + __m256i s1 = _mm256_loadu_si256((const __m256i*)src_argb1); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_subs_epu8(s0, s1)); + src_argb += 32; + src_argb1 += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBSUBTRACTROW_AVX2 + +#ifdef HAS_ARGBCOPYALPHAROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBCopyAlphaRow_AVX2(const uint8_t* src, uint8_t* dst, int width) { + const __m256i mask_alpha = _mm256_set1_epi32((int)0xff000000u); + do { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src); + __m256i s1 = _mm256_loadu_si256((const __m256i*)(src + 32)); + __m256i d0 = _mm256_loadu_si256((const __m256i*)dst); + __m256i d1 = _mm256_loadu_si256((const __m256i*)(dst + 32)); + _mm256_storeu_si256((__m256i*)dst, _mm256_blendv_epi8(d0, s0, mask_alpha)); + _mm256_storeu_si256((__m256i*)(dst + 32), + _mm256_blendv_epi8(d1, s1, mask_alpha)); + src += 64; + dst += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBCOPYALPHAROW_AVX2 + +#ifdef HAS_ARGBPOLYNOMIALROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBPolynomialRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + const float* poly, + int width) { + const __m256 c0 = _mm256_broadcast_ps((const __m128*)(poly + 0)); + const __m256 c1 = _mm256_broadcast_ps((const __m128*)(poly + 4)); + const __m256 c2 = _mm256_broadcast_ps((const __m128*)(poly + 8)); + const __m256 c3 = _mm256_broadcast_ps((const __m128*)(poly + 12)); + do { + __m128i s8 = _mm_loadl_epi64((const __m128i*)src_argb); + src_argb += 8; + __m256 x = _mm256_cvtepi32_ps(_mm256_cvtepu8_epi32(s8)); + __m256 x2 = _mm256_mul_ps(x, x); + __m256 res = _mm256_add_ps(c0, _mm256_mul_ps(c1, x)); + res = _mm256_add_ps(res, _mm256_mul_ps(c2, x2)); + res = _mm256_add_ps(res, _mm256_mul_ps(_mm256_mul_ps(c3, x), x2)); + __m256i i32 = _mm256_cvttps_epi32(res); + __m256i u16 = + _mm256_permute4x64_epi64(_mm256_packus_epi32(i32, i32), 0xd8); + __m128i u8 = _mm_packus_epi16(_mm256_castsi256_si128(u16), + _mm256_castsi256_si128(u16)); + _mm_storel_epi64((__m128i*)dst_argb, u8); + dst_argb += 8; + width -= 2; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBPOLYNOMIALROW_AVX2 + +#ifdef HAS_ARGBUNATTENUATEROW_AVX2 +LIBYUV_TARGET_AVX2 +void ARGBUnattenuateRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint8_t kUnattenShuffleAlpha[16] = { + 0u, 1u, 0u, 1u, 0u, 1u, 6u, 7u, 8u, 9u, 8u, 9u, 8u, 9u, 14u, 15u}; + const __m256i ymm_shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kUnattenShuffleAlpha)); + do { + __m256i ymm3 = _mm256_setr_epi32( + (int)fixed_invtbl8[src_argb[3]], (int)fixed_invtbl8[src_argb[7]], + (int)fixed_invtbl8[src_argb[11]], (int)fixed_invtbl8[src_argb[15]], + (int)fixed_invtbl8[src_argb[19]], (int)fixed_invtbl8[src_argb[23]], + (int)fixed_invtbl8[src_argb[27]], (int)fixed_invtbl8[src_argb[31]]); + __m256i ymm6 = _mm256_loadu_si256((const __m256i*)src_argb); + __m256i ymm0 = _mm256_unpacklo_epi8(ymm6, ymm6); + __m256i ymm1 = _mm256_unpackhi_epi8(ymm6, ymm6); + __m256i ymm2 = + _mm256_shuffle_epi8(_mm256_unpacklo_epi16(ymm3, ymm3), ymm_shuf); + ymm3 = _mm256_shuffle_epi8(_mm256_unpackhi_epi16(ymm3, ymm3), ymm_shuf); + ymm0 = _mm256_mulhi_epu16(ymm0, ymm2); + ymm1 = _mm256_mulhi_epu16(ymm1, ymm3); + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_packus_epi16(ymm0, ymm1)); + src_argb += 32; + dst_argb += 32; + width -= 8; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_ARGBUNATTENUATEROW_AVX2 + +#ifdef HAS_COMPUTECUMULATIVESUMROW_AVX2 +LIBYUV_TARGET_AVX2 +void ComputeCumulativeSumRow_AVX2(const uint8_t* row, + int32_t* cumsum, + const int32_t* previous_cumsum, + int width) { + static const uint8_t kCumSumTranspose[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; + static const int8_t kCumSumMaddLo[32] = { + 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, + 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0}; + static const int8_t kCumSumMaddHi[32] = { + 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m256i ymm0 = _mm256_setzero_si256(); + const __m256i ymm5 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kCumSumTranspose)); + const __m256i ymm6 = _mm256_loadu_si256((const __m256i*)kCumSumMaddLo); + const __m256i ymm7 = _mm256_loadu_si256((const __m256i*)kCumSumMaddHi); + const __m256i ymm8 = _mm256_set1_epi16(1); + + while (width >= 4) { + __m256i ymm2 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)row)); + row += 16; + ymm2 = _mm256_shuffle_epi8(ymm2, ymm5); + __m256i ymm3 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm2, ymm6), ymm8); + __m256i ymm4 = _mm256_madd_epi16(_mm256_maddubs_epi16(ymm2, ymm7), ymm8); + __m256i sum4 = _mm256_permute2x128_si256(ymm4, ymm4, 0x11); + ymm3 = _mm256_add_epi32( + _mm256_add_epi32(ymm3, + _mm256_loadu_si256((const __m256i*)previous_cumsum)), + ymm0); + ymm4 = _mm256_add_epi32( + _mm256_add_epi32( + ymm4, _mm256_loadu_si256((const __m256i*)(previous_cumsum + 8))), + ymm0); + previous_cumsum += 16; + ymm0 = _mm256_add_epi32(ymm0, sum4); + _mm256_storeu_si256((__m256i*)cumsum, ymm3); + _mm256_storeu_si256((__m256i*)(cumsum + 8), ymm4); + cumsum += 16; + width -= 4; + } + + __m128i xmm0 = _mm256_castsi256_si128(ymm0); + while (width > 0) { + int32_t px; + memcpy(&px, row, sizeof(px)); + row += 4; + __m128i xmm2 = _mm_cvtepu8_epi32(_mm_cvtsi32_si128(px)); + xmm0 = _mm_add_epi32(xmm0, xmm2); + xmm2 = _mm_add_epi32(xmm0, + _mm_loadu_si128((const __m128i*)previous_cumsum)); + previous_cumsum += 4; + _mm_storeu_si128((__m128i*)cumsum, xmm2); + cumsum += 4; + --width; + } + _mm256_zeroupper(); +} +#endif // HAS_COMPUTECUMULATIVESUMROW_AVX2 + +#ifdef HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 +LIBYUV_TARGET_AVX2 +void CumulativeSumToAverageRow_AVX2(const int32_t* topleft, + const int32_t* botleft, + int width, + int area, + uint8_t* dst, + int count) { + __m128 xmm5 = _mm_cvtsi32_ss(_mm_setzero_ps(), area); + __m128 xmm4 = _mm_rcp_ss(xmm5); + xmm4 = _mm_shuffle_ps(xmm4, xmm4, 0); + + if (count >= 4) { + __m256 ymm4 = _mm256_insertf128_ps(_mm256_castps128_ps256(xmm4), xmm4, 1); + if ((uint32_t)area <= 0x80u) { + __m128 xmm6 = _mm_cvtepi32_ps(_mm_set1_epi32(0x0000ffff)); + xmm6 = _mm_mul_ss(_mm_add_ss(xmm5, xmm6), xmm4); + __m128i i6 = _mm_cvtps_epi32(xmm6); + i6 = _mm_packs_epi32(i6, i6); + __m256i ymm6 = _mm256_broadcastw_epi16(i6); + while (count >= 4) { + __m256i ymm0 = _mm256_sub_epi32( + _mm256_loadu_si256((const __m256i*)topleft), + _mm256_loadu_si256((const __m256i*)(topleft + width))); + __m256i ymm1 = _mm256_sub_epi32( + _mm256_loadu_si256((const __m256i*)(topleft + 8)), + _mm256_loadu_si256((const __m256i*)(topleft + width + 8))); + topleft += 16; + ymm0 = _mm256_sub_epi32(ymm0, + _mm256_loadu_si256((const __m256i*)botleft)); + ymm1 = _mm256_sub_epi32( + ymm1, _mm256_loadu_si256((const __m256i*)(botleft + 8))); + ymm0 = _mm256_add_epi32( + ymm0, _mm256_loadu_si256((const __m256i*)(botleft + width))); + ymm1 = _mm256_add_epi32( + ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8))); + botleft += 16; + ymm0 = _mm256_mulhi_epu16(_mm256_packs_epi32(ymm0, ymm1), ymm6); + __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0), + _mm256_extracti128_si256(ymm0, 1)); + _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8)); + dst += 16; + count -= 4; + } + } else { + while (count >= 4) { + __m256i ymm0 = _mm256_sub_epi32( + _mm256_loadu_si256((const __m256i*)topleft), + _mm256_loadu_si256((const __m256i*)(topleft + width))); + __m256i ymm1 = _mm256_sub_epi32( + _mm256_loadu_si256((const __m256i*)(topleft + 8)), + _mm256_loadu_si256((const __m256i*)(topleft + width + 8))); + topleft += 16; + ymm0 = _mm256_sub_epi32(ymm0, + _mm256_loadu_si256((const __m256i*)botleft)); + ymm1 = _mm256_sub_epi32( + ymm1, _mm256_loadu_si256((const __m256i*)(botleft + 8))); + ymm0 = _mm256_add_epi32( + ymm0, _mm256_loadu_si256((const __m256i*)(botleft + width))); + ymm1 = _mm256_add_epi32( + ymm1, _mm256_loadu_si256((const __m256i*)(botleft + width + 8))); + botleft += 16; + ymm0 = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm0), ymm4)); + ymm1 = _mm256_cvtps_epi32(_mm256_mul_ps(_mm256_cvtepi32_ps(ymm1), ymm4)); + ymm0 = _mm256_packs_epi32(ymm0, ymm1); + __m128i out = _mm_packus_epi16(_mm256_castsi256_si128(ymm0), + _mm256_extracti128_si256(ymm0, 1)); + _mm_storeu_si128((__m128i*)dst, _mm_shuffle_epi32(out, 0xd8)); + dst += 16; + count -= 4; + } + } + _mm256_zeroupper(); + } + + while (count > 0) { + __m128i d0 = _mm_sub_epi32( + _mm_loadu_si128((const __m128i*)topleft), + _mm_loadu_si128((const __m128i*)(topleft + width))); + topleft += 4; + __m128i d1 = _mm_sub_epi32( + _mm_loadu_si128((const __m128i*)(botleft + width)), + _mm_loadu_si128((const __m128i*)botleft)); + botleft += 4; + d0 = _mm_add_epi32(d0, d1); + d0 = _mm_cvtps_epi32(_mm_mul_ps(_mm_cvtepi32_ps(d0), xmm4)); + d0 = _mm_packs_epi32(d0, d0); + d0 = _mm_packus_epi16(d0, d0); + int32_t out32 = _mm_cvtsi128_si32(d0); + memcpy(dst, &out32, sizeof(out32)); + dst += 4; + --count; + } +} +#endif // HAS_CUMULATIVESUMTOAVERAGEROW_AVX2 + +#ifdef HAS_I422TOYUY2ROW_AVX2 +LIBYUV_TARGET_AVX2 +void I422ToYUY2Row_AVX2(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_yuy2, + int width) { + do { + __m128i u = _mm_loadu_si128((const __m128i*)src_u); + __m128i v = _mm_loadu_si128((const __m128i*)src_v); + __m256i uv = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(u, v)), + _mm_unpackhi_epi8(u, v), 1); + __m256i y = _mm256_loadu_si256((const __m256i*)src_y); + __m256i lo = _mm256_unpacklo_epi8(y, uv); + __m256i hi = _mm256_unpackhi_epi8(y, uv); + _mm256_storeu_si256((__m256i*)dst_yuy2, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_yuy2 + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_y += 32; + src_u += 16; + src_v += 16; + dst_yuy2 += 64; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_I422TOYUY2ROW_AVX2 + +#ifdef HAS_I422TOUYVYROW_AVX2 +LIBYUV_TARGET_AVX2 +void I422ToUYVYRow_AVX2(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uyvy, + int width) { + do { + __m128i u = _mm_loadu_si128((const __m128i*)src_u); + __m128i v = _mm_loadu_si128((const __m128i*)src_v); + __m256i uv = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(u, v)), + _mm_unpackhi_epi8(u, v), 1); + __m256i y = _mm256_loadu_si256((const __m256i*)src_y); + __m256i lo = _mm256_unpacklo_epi8(uv, y); + __m256i hi = _mm256_unpackhi_epi8(uv, y); + _mm256_storeu_si256((__m256i*)dst_uyvy, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_uyvy + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_y += 32; + src_u += 16; + src_v += 16; + dst_uyvy += 64; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_I422TOUYVYROW_AVX2 + +#ifdef HAS_MERGEARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeARGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + const uint8_t* src_a, + uint8_t* dst_argb, + int width) { + do { + __m128i r = _mm_loadu_si128((const __m128i*)src_r); + __m128i g = _mm_loadu_si128((const __m128i*)src_g); + __m128i b = _mm_loadu_si128((const __m128i*)src_b); + __m128i a = _mm_loadu_si128((const __m128i*)src_a); + __m256i bg = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)), + _mm_unpackhi_epi8(b, g), 1); + __m256i ra = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)), + _mm_unpackhi_epi8(r, a), 1); + __m256i lo = _mm256_unpacklo_epi16(bg, ra); + __m256i hi = _mm256_unpackhi_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEARGBROW_AVX2 + +#ifdef HAS_MERGEXRGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeXRGBRow_AVX2(const uint8_t* src_r, + const uint8_t* src_g, + const uint8_t* src_b, + uint8_t* dst_argb, + int width) { + const __m128i a = _mm_set1_epi8((char)0xff); + do { + __m128i r = _mm_loadu_si128((const __m128i*)src_r); + __m128i g = _mm_loadu_si128((const __m128i*)src_g); + __m128i b = _mm_loadu_si128((const __m128i*)src_b); + __m256i bg = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)), + _mm_unpackhi_epi8(b, g), 1); + __m256i ra = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)), + _mm_unpackhi_epi8(r, a), 1); + __m256i lo = _mm256_unpacklo_epi16(bg, ra); + __m256i hi = _mm256_unpackhi_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEXRGBROW_AVX2 + +#ifdef HAS_MERGEARGB16TO8ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeARGB16To8Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + const uint16_t* src_a, + uint8_t* dst_argb, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(depth - 8); + const __m256i ymm_255 = _mm256_set1_epi16(255); + do { + __m256i r16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), shift128), + ymm_255); + __m256i g16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), shift128), + ymm_255); + __m256i b16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), shift128), + ymm_255); + __m256i a16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_a), shift128), + ymm_255); + __m256i br8 = + _mm256_permute4x64_epi64(_mm256_packus_epi16(b16, r16), 0xd8); + __m256i ga8 = + _mm256_permute4x64_epi64(_mm256_packus_epi16(g16, a16), 0xd8); + __m128i b = _mm256_castsi256_si128(br8); + __m128i r = _mm256_extracti128_si256(br8, 1); + __m128i g = _mm256_castsi256_si128(ga8); + __m128i a = _mm256_extracti128_si256(ga8, 1); + __m256i bg = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)), + _mm_unpackhi_epi8(b, g), 1); + __m256i ra = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)), + _mm_unpackhi_epi8(r, a), 1); + __m256i lo = _mm256_unpacklo_epi16(bg, ra); + __m256i hi = _mm256_unpackhi_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + src_a += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEARGB16TO8ROW_AVX2 + +#ifdef HAS_MERGEXRGB16TO8ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeXRGB16To8Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint8_t* dst_argb, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(depth - 8); + const __m256i ymm_255 = _mm256_set1_epi16(255); + const __m128i a = _mm_set1_epi8((char)0xff); + do { + __m256i r16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), shift128), + ymm_255); + __m256i g16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), shift128), + ymm_255); + __m256i b16 = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), shift128), + ymm_255); + __m256i br8 = + _mm256_permute4x64_epi64(_mm256_packus_epi16(b16, r16), 0xd8); + __m256i g8 = + _mm256_permute4x64_epi64(_mm256_packus_epi16(g16, g16), 0xd8); + __m128i b = _mm256_castsi256_si128(br8); + __m128i r = _mm256_extracti128_si256(br8, 1); + __m128i g = _mm256_castsi256_si128(g8); + __m256i bg = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(b, g)), + _mm_unpackhi_epi8(b, g), 1); + __m256i ra = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(r, a)), + _mm_unpackhi_epi8(r, a), 1); + __m256i lo = _mm256_unpacklo_epi16(bg, ra); + __m256i hi = _mm256_unpackhi_epi16(bg, ra); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_permute2x128_si256(lo, hi, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_permute2x128_si256(lo, hi, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_argb += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEXRGB16TO8ROW_AVX2 + +#ifdef HAS_MERGEAR64ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeAR64Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + const uint16_t* src_a, + uint16_t* dst_ar64, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(16 - depth); + const __m256i ymm_max = + _mm256_set1_epi16((short)((1u << depth) - 1u)); + do { + __m256i r = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max), + shift128); + __m256i g = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_g), ymm_max), + shift128); + __m256i b = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_b), ymm_max), + shift128); + __m256i a = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_a), ymm_max), + shift128); + __m256i bg_lo = _mm256_unpacklo_epi16(b, g); + __m256i bg_hi = _mm256_unpackhi_epi16(b, g); + __m256i ra_lo = _mm256_unpacklo_epi16(r, a); + __m256i ra_hi = _mm256_unpackhi_epi16(r, a); + __m256i p0_8 = _mm256_unpacklo_epi32(bg_lo, ra_lo); + __m256i p2_10 = _mm256_unpackhi_epi32(bg_lo, ra_lo); + __m256i p4_12 = _mm256_unpacklo_epi32(bg_hi, ra_hi); + __m256i p6_14 = _mm256_unpackhi_epi32(bg_hi, ra_hi); + _mm256_storeu_si256((__m256i*)dst_ar64, + _mm256_permute2x128_si256(p0_8, p2_10, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 16), + _mm256_permute2x128_si256(p4_12, p6_14, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 32), + _mm256_permute2x128_si256(p0_8, p2_10, 0x31)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 48), + _mm256_permute2x128_si256(p4_12, p6_14, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + src_a += 16; + dst_ar64 += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEAR64ROW_AVX2 + +#ifdef HAS_MERGEXR64ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeXR64Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint16_t* dst_ar64, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(16 - depth); + const __m256i ymm_max = + _mm256_set1_epi16((short)((1u << depth) - 1u)); + const __m256i a = _mm256_set1_epi16((short)0xffff); + do { + __m256i r = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_r), ymm_max), + shift128); + __m256i g = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_g), ymm_max), + shift128); + __m256i b = _mm256_sll_epi16( + _mm256_min_epu16(_mm256_loadu_si256((const __m256i*)src_b), ymm_max), + shift128); + __m256i bg_lo = _mm256_unpacklo_epi16(b, g); + __m256i bg_hi = _mm256_unpackhi_epi16(b, g); + __m256i ra_lo = _mm256_unpacklo_epi16(r, a); + __m256i ra_hi = _mm256_unpackhi_epi16(r, a); + __m256i p0_8 = _mm256_unpacklo_epi32(bg_lo, ra_lo); + __m256i p2_10 = _mm256_unpackhi_epi32(bg_lo, ra_lo); + __m256i p4_12 = _mm256_unpacklo_epi32(bg_hi, ra_hi); + __m256i p6_14 = _mm256_unpackhi_epi32(bg_hi, ra_hi); + _mm256_storeu_si256((__m256i*)dst_ar64, + _mm256_permute2x128_si256(p0_8, p2_10, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 16), + _mm256_permute2x128_si256(p4_12, p6_14, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 32), + _mm256_permute2x128_si256(p0_8, p2_10, 0x31)); + _mm256_storeu_si256((__m256i*)(dst_ar64 + 48), + _mm256_permute2x128_si256(p4_12, p6_14, 0x31)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_ar64 += 64; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_MERGEXR64ROW_AVX2 + +#ifdef HAS_NV21TOYUV24ROW_AVX2 +LIBYUV_TARGET_AVX2 +void NV21ToYUV24Row_AVX2(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_yuv24, + int width) { + static const uint8_t kYUV24Shuffle[3][16] = { + {8, 9, 0, 8, 9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12}, + {9, 1, 10, 11, 2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15}, + {2, 10, 11, 3, 12, 13, 4, 12, 13, 5, 14, 15, 6, 14, 15, 7}}; + const __m256i ymm4 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kYUV24Shuffle[0])); + const __m256i ymm5 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kYUV24Shuffle[1])); + const __m256i ymm6 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kYUV24Shuffle[2])); + + do { + __m256 ymm2 = _mm256_castsi256_ps(_mm256_loadu_si256((const __m256i*)src_y)); + __m256 ymm3 = + _mm256_castsi256_ps(_mm256_loadu_si256((const __m256i*)src_vu)); + __m256i c0 = _mm256_shuffle_epi8( + _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0x44)), ymm4); + __m256i c1 = _mm256_shuffle_epi8( + _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0x99)), ymm5); + __m256i c2 = _mm256_shuffle_epi8( + _mm256_castps_si256(_mm256_shuffle_ps(ymm2, ymm3, 0xee)), ymm6); + _mm256_storeu_si256((__m256i*)dst_yuv24, + _mm256_permute2x128_si256(c0, c1, 0x20)); + _mm256_storeu_si256((__m256i*)(dst_yuv24 + 32), + _mm256_permute2x128_si256(c2, c0, 0x30)); + _mm256_storeu_si256((__m256i*)(dst_yuv24 + 64), + _mm256_permute2x128_si256(c1, c2, 0x31)); + src_y += 32; + src_vu += 32; + dst_yuv24 += 96; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_NV21TOYUV24ROW_AVX2 + +#ifdef HAS_SPLITARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void SplitARGBRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + uint8_t* dst_a, + int width) { + static const uint8_t kShufSplitARGB[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufSplitARGB)); + do { + __m256i t0 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)src_argb), shuf); + __m256i t1 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_argb + 32)), shuf); + __m256i p0 = _mm256_permute2x128_si256(t0, t1, 0x20); + __m256i p1 = _mm256_permute2x128_si256(t0, t1, 0x31); + __m256i bg = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(p0, p1), 0xd8); + __m256i ra = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(p0, p1), 0xd8); + _mm_storeu_si128((__m128i*)dst_b, _mm256_castsi256_si128(bg)); + _mm_storeu_si128((__m128i*)dst_g, _mm256_extracti128_si256(bg, 1)); + _mm_storeu_si128((__m128i*)dst_r, _mm256_castsi256_si128(ra)); + _mm_storeu_si128((__m128i*)dst_a, _mm256_extracti128_si256(ra, 1)); + src_argb += 64; + dst_r += 16; + dst_g += 16; + dst_b += 16; + dst_a += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITARGBROW_AVX2 + +#ifdef HAS_SPLITXRGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void SplitXRGBRow_AVX2(const uint8_t* src_argb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width) { + static const uint8_t kShufSplitARGB[16] = {0, 4, 8, 12, 1, 5, 9, 13, + 2, 6, 10, 14, 3, 7, 11, 15}; + const __m256i shuf = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kShufSplitARGB)); + do { + __m256i t0 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)src_argb), shuf); + __m256i t1 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_argb + 32)), shuf); + __m256i p0 = _mm256_permute2x128_si256(t0, t1, 0x20); + __m256i p1 = _mm256_permute2x128_si256(t0, t1, 0x31); + __m256i bg = _mm256_permute4x64_epi64(_mm256_unpacklo_epi32(p0, p1), 0xd8); + __m256i ra = _mm256_permute4x64_epi64(_mm256_unpackhi_epi32(p0, p1), 0xd8); + _mm_storeu_si128((__m128i*)dst_b, _mm256_castsi256_si128(bg)); + _mm_storeu_si128((__m128i*)dst_g, _mm256_extracti128_si256(bg, 1)); + _mm_storeu_si128((__m128i*)dst_r, _mm256_castsi256_si128(ra)); + src_argb += 64; + dst_r += 16; + dst_g += 16; + dst_b += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITXRGBROW_AVX2 + +#ifdef HAS_SPLITRGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void SplitRGBRow_AVX2(const uint8_t* src_rgb, + uint8_t* dst_r, + uint8_t* dst_g, + uint8_t* dst_b, + int width) { + static const uint8_t kSplitRGBShuffle[5][16] = { + {0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u}, + {1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u, 2u, 5u, 8u, 11u, 14u}, + {2u, 5u, 8u, 11u, 14u, 1u, 4u, 7u, 10u, 13u, 0u, 3u, 6u, 9u, 12u, 15u}, + {0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, + 0u}, + {0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, 0u, 0u, 128u, + 0u}}; + const __m256i s0 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[0])); + const __m256i s1 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[1])); + const __m256i s2 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[2])); + const __m256i m0 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[3])); + const __m256i m1 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)kSplitRGBShuffle[4])); + + do { + __m256i ymm4 = _mm256_loadu_si256((const __m256i*)src_rgb); + __m256i ymm5 = _mm256_loadu_si256((const __m256i*)(src_rgb + 32)); + __m256i ymm6 = _mm256_loadu_si256((const __m256i*)(src_rgb + 64)); + src_rgb += 96; + __m256i ymm1 = _mm256_blend_epi32(ymm4, ymm5, 0xf0); + __m256i ymm2 = _mm256_permute2x128_si256(ymm4, ymm6, 0x21); + __m256i ymm3 = _mm256_blend_epi32(ymm5, ymm6, 0xf0); + ymm4 = _mm256_blendv_epi8(ymm1, ymm3, m0); + ymm5 = _mm256_blendv_epi8(ymm2, ymm1, m0); + ymm6 = _mm256_blendv_epi8(ymm3, ymm2, m0); + ymm1 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm4, ymm5, m1), s0); + ymm2 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm5, ymm6, m1), s1); + ymm3 = _mm256_shuffle_epi8(_mm256_blendv_epi8(ymm6, ymm4, m1), s2); + _mm256_storeu_si256((__m256i*)dst_r, ymm1); + _mm256_storeu_si256((__m256i*)dst_g, ymm2); + _mm256_storeu_si256((__m256i*)dst_b, ymm3); + dst_r += 32; + dst_g += 32; + dst_b += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITRGBROW_AVX2 + +#ifdef HAS_SPLITUVROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void SplitUVRow_16_AVX2(const uint16_t* src_uv, + uint16_t* dst_u, + uint16_t* dst_v, + int depth, + int width) { + const __m128i shift128 = _mm_cvtsi32_si128(16 - depth); + const __m256i mask16 = _mm256_set1_epi32(0x0000ffff); + do { + __m256i uv0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i uv1 = _mm256_loadu_si256((const __m256i*)(src_uv + 16)); + __m256i u = _mm256_permute4x64_epi64( + _mm256_packus_epi32(_mm256_and_si256(uv0, mask16), + _mm256_and_si256(uv1, mask16)), + 0xd8); + __m256i v = _mm256_permute4x64_epi64( + _mm256_packus_epi32(_mm256_srli_epi32(uv0, 16), + _mm256_srli_epi32(uv1, 16)), + 0xd8); + _mm256_storeu_si256((__m256i*)dst_u, _mm256_srl_epi16(u, shift128)); + _mm256_storeu_si256((__m256i*)dst_v, _mm256_srl_epi16(v, shift128)); + src_uv += 32; + dst_u += 16; + dst_v += 16; + width -= 16; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_SPLITUVROW_16_AVX2 + +#ifdef HAS_YUY2TOYROW_AVX2 +LIBYUV_TARGET_AVX2 +void YUY2ToYRow_AVX2(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i s0 = + _mm256_and_si256(_mm256_loadu_si256((const __m256i*)src_yuy2), + mask_00ff); + __m256i s1 = + _mm256_and_si256(_mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)), + mask_00ff); + _mm256_storeu_si256( + (__m256i*)dst_y, + _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8)); + src_yuy2 += 64; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOYROW_AVX2 + +#ifdef HAS_UYVYTOYROW_AVX2 +LIBYUV_TARGET_AVX2 +void UYVYToYRow_AVX2(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + do { + __m256i s0 = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)src_uyvy), 8); + __m256i s1 = + _mm256_srli_epi16(_mm256_loadu_si256((const __m256i*)(src_uyvy + 32)), + 8); + _mm256_storeu_si256( + (__m256i*)dst_y, + _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8)); + src_uyvy += 64; + dst_y += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_UYVYTOYROW_AVX2 + +#ifdef HAS_YUY2TOUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i y0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_yuy2), + _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2))); + __m256i y1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)), + _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2 + 32))); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)), + 0xd8); + __m256i u = _mm256_and_si256(uv, mask_00ff); + __m256i v = _mm256_srli_epi16(uv, 8); + u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8); + v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v)); + src_yuy2 += 64; + dst_u += 16; + dst_v += 16; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOUVROW_AVX2 + +#ifdef HAS_YUY2TOUV422ROW_AVX2 +LIBYUV_TARGET_AVX2 +void YUY2ToUV422Row_AVX2(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i y0 = _mm256_loadu_si256((const __m256i*)src_yuy2); + __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)), + 0xd8); + __m256i u = _mm256_and_si256(uv, mask_00ff); + __m256i v = _mm256_srli_epi16(uv, 8); + u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8); + v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v)); + src_yuy2 += 64; + dst_u += 16; + dst_v += 16; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_YUY2TOUV422ROW_AVX2 + +#ifdef HAS_YUY2TONVUVROW_AVX2 +LIBYUV_TARGET_AVX2 +void YUY2ToNVUVRow_AVX2(const uint8_t* src_yuy2, + int stride_yuy2, + uint8_t* dst_uv, + int width) { + do { + __m256i y0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_yuy2), + _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2))); + __m256i y1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_yuy2 + 32)), + _mm256_loadu_si256((const __m256i*)(src_yuy2 + stride_yuy2 + 32))); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_srli_epi16(y0, 8), _mm256_srli_epi16(y1, 8)), + 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, uv); + src_yuy2 += 64; + dst_uv += 32; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // HAS_YUY2TONVUVROW_AVX2 + +#if defined(HAS_UYVYTOUVROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2) +LIBYUV_TARGET_AVX2 +void UYVYToUVRow_AVX2(const uint8_t* src_uyvy, + int stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i y0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_uyvy), + _mm256_loadu_si256((const __m256i*)(src_uyvy + stride_uyvy))); + __m256i y1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_uyvy + 32)), + _mm256_loadu_si256((const __m256i*)(src_uyvy + stride_uyvy + 32))); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_and_si256(y0, mask_00ff), + _mm256_and_si256(y1, mask_00ff)), + 0xd8); + __m256i u = _mm256_and_si256(uv, mask_00ff); + __m256i v = _mm256_srli_epi16(uv, 8); + u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8); + v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v)); + src_uyvy += 64; + dst_u += 16; + dst_v += 16; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // defined(HAS_UYVYTOUVROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2) + +#if defined(HAS_UYVYTOUV422ROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2) +LIBYUV_TARGET_AVX2 +void UYVYToUV422Row_AVX2(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const __m256i mask_00ff = _mm256_set1_epi16(0x00ff); + do { + __m256i y0 = _mm256_loadu_si256((const __m256i*)src_uyvy); + __m256i y1 = _mm256_loadu_si256((const __m256i*)(src_uyvy + 32)); + __m256i uv = _mm256_permute4x64_epi64( + _mm256_packus_epi16(_mm256_and_si256(y0, mask_00ff), + _mm256_and_si256(y1, mask_00ff)), + 0xd8); + __m256i u = _mm256_and_si256(uv, mask_00ff); + __m256i v = _mm256_srli_epi16(uv, 8); + u = _mm256_permute4x64_epi64(_mm256_packus_epi16(u, u), 0xd8); + v = _mm256_permute4x64_epi64(_mm256_packus_epi16(v, v), 0xd8); + _mm_storeu_si128((__m128i*)dst_u, _mm256_castsi256_si128(u)); + _mm_storeu_si128((__m128i*)dst_v, _mm256_castsi256_si128(v)); + src_uyvy += 64; + dst_u += 16; + dst_v += 16; + width -= 32; + } while (width > 0); + _mm256_zeroupper(); +} +#endif // defined(HAS_UYVYTOUV422ROW_AVX2) || defined(HAS_UYVYTOYROW_AVX2) + +#ifdef HAS_I444TOARGBROW_AVX2 +LIBYUV_TARGET_AVX2 +void I444ToARGBRow_AVX2(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + const __m256i ymm_kUVToB = + _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToB); + const __m256i ymm_kUVToG = + _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToG); + const __m256i ymm_kUVToR = + _mm256_loadu_si256((const __m256i*)yuvconstants->kUVToR); + const __m256i ymm_kYToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYToRgb); + const __m256i ymm_kYBiasToRgb = + _mm256_loadu_si256((const __m256i*)yuvconstants->kYBiasToRgb); + const __m256i ymm_128 = _mm256_set1_epi8((char)0x80); + const __m256i ymm_alpha = _mm256_set1_epi8((char)0xff); + + while (width >= 16) { + __m128i xmm_u = _mm_loadu_si128((const __m128i*)u_buf); + __m128i xmm_v = _mm_loadu_si128((const __m128i*)v_buf); + __m128i xmm_y = _mm_loadu_si128((const __m128i*)y_buf); + u_buf += 16; + v_buf += 16; + y_buf += 16; + + __m256i ymm3 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(xmm_u, xmm_v)), + _mm_unpackhi_epi8(xmm_u, xmm_v), 1); + __m256i ymm4 = _mm256_inserti128_si256( + _mm256_castsi128_si256(_mm_unpacklo_epi8(xmm_y, xmm_y)), + _mm_unpackhi_epi8(xmm_y, xmm_y), 1); + + ymm3 = _mm256_sub_epi8(ymm3, ymm_128); + ymm4 = _mm256_mulhi_epu16(ymm4, ymm_kYToRgb); + + __m256i ymm0 = _mm256_maddubs_epi16(ymm_kUVToB, ymm3); + __m256i ymm1 = _mm256_maddubs_epi16(ymm_kUVToG, ymm3); + __m256i ymm2 = _mm256_maddubs_epi16(ymm_kUVToR, ymm3); + + ymm4 = _mm256_add_epi16(ymm4, ymm_kYBiasToRgb); + + ymm0 = _mm256_srai_epi16(_mm256_adds_epi16(ymm0, ymm4), 6); + ymm1 = _mm256_srai_epi16(_mm256_subs_epi16(ymm4, ymm1), 6); + ymm2 = _mm256_srai_epi16(_mm256_adds_epi16(ymm2, ymm4), 6); + + ymm0 = _mm256_packus_epi16(ymm0, ymm0); + ymm1 = _mm256_packus_epi16(ymm1, ymm1); + ymm2 = _mm256_packus_epi16(ymm2, ymm2); + + ymm0 = _mm256_permute4x64_epi64(_mm256_unpacklo_epi8(ymm0, ymm1), 0xd8); + ymm2 = + _mm256_permute4x64_epi64(_mm256_unpacklo_epi8(ymm2, ymm_alpha), 0xd8); + + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi16(ymm0, ymm2)); + _mm256_storeu_si256((__m256i*)(dst_argb + 32), + _mm256_unpackhi_epi16(ymm0, ymm2)); + + dst_argb += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_I444TOARGBROW_AVX2 + +#ifdef HAS_MERGEXR30ROW_AVX2 +LIBYUV_TARGET_AVX2 +void MergeXR30Row_AVX2(const uint16_t* src_r, + const uint16_t* src_g, + const uint16_t* src_b, + uint8_t* dst_ar30, + int depth, + int width) { + int shift = depth - 10; + __m128i v_shift = _mm_cvtsi32_si128(shift); + __m256i v_alpha = _mm256_set1_epi16(0x0030); + __m256i v_max = _mm256_set1_epi16(1023); + while (width > 0) { + __m256i r = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_r), v_shift), + v_max); + __m256i g = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_g), v_shift), + v_max); + __m256i b = _mm256_min_epu16( + _mm256_srl_epi16(_mm256_loadu_si256((const __m256i*)src_b), v_shift), + v_max); + r = _mm256_slli_epi16(_mm256_permute4x64_epi64(r, 0xd8), 4); + g = _mm256_permute4x64_epi64(g, 0xd8); + b = _mm256_permute4x64_epi64(b, 0xd8); + __m256i rb_lo = _mm256_unpacklo_epi16(b, r); + __m256i rb_hi = _mm256_unpackhi_epi16(b, r); + __m256i ag_lo = _mm256_slli_epi32(_mm256_unpacklo_epi16(g, v_alpha), 10); + __m256i ag_hi = _mm256_slli_epi32(_mm256_unpackhi_epi16(g, v_alpha), 10); + _mm256_storeu_si256((__m256i*)dst_ar30, _mm256_or_si256(rb_lo, ag_lo)); + _mm256_storeu_si256((__m256i*)(dst_ar30 + 32), + _mm256_or_si256(rb_hi, ag_hi)); + src_r += 16; + src_g += 16; + src_b += 16; + dst_ar30 += 64; + width -= 16; + } + _mm256_zeroupper(); +} +#endif // HAS_MERGEXR30ROW_AVX2 + +#ifdef HAS_MERGEUVROW_16_AVX2 +LIBYUV_TARGET_AVX2 +void MergeUVRow_16_AVX2(const uint16_t* src_u, + const uint16_t* src_v, + uint16_t* dst_uv, + int depth, + int width) { + const __m128i sh_u = _mm_cvtsi32_si128(16 - depth); + const __m128i sh_v = _mm_cvtsi32_si128(32 - depth); + while (width > 0) { + __m256i u32 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_u)); + __m256i v32 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_v)); + src_u += 8; + src_v += 8; + u32 = _mm256_sll_epi16(u32, sh_u); + v32 = _mm256_sll_epi32(v32, sh_v); + _mm256_storeu_si256((__m256i*)dst_uv, _mm256_or_si256(u32, v32)); + dst_uv += 16; + width -= 8; + } + _mm256_zeroupper(); +} +#endif // HAS_MERGEUVROW_16_AVX2 + #ifdef __cplusplus } // extern "C" } // namespace libyuv diff --git a/source/scale.cc b/source/scale.cc index a9fdf79c8..f2157409c 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -206,6 +206,20 @@ static void ScalePlaneDown2_16(int src_width, src_stride = 0; } +#if defined(HAS_SCALEROWDOWN2_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleRowDown2 = + filtering == kFilterNone + ? ScaleRowDown2_16_Any_AVX2 + : (filtering == kFilterLinear ? ScaleRowDown2Linear_16_Any_AVX2 + : ScaleRowDown2Box_16_Any_AVX2); + if (IS_ALIGNED(dst_width, 16)) { + ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_AVX2 + : filtering == kFilterLinear ? ScaleRowDown2Linear_16_AVX2 + : ScaleRowDown2Box_16_AVX2; + } + } +#endif #if defined(HAS_SCALEROWDOWN2_16_NEON) if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 16)) { ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_NEON @@ -220,15 +234,6 @@ static void ScalePlaneDown2_16(int src_width, : ScaleRowDown2Box_16_SME; } #endif -#if defined(HAS_SCALEROWDOWN2_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 16)) { - ScaleRowDown2 = - filtering == kFilterNone - ? ScaleRowDown2_16_SSE2 - : (filtering == kFilterLinear ? ScaleRowDown2Linear_16_SSE2 - : ScaleRowDown2Box_16_SSE2); - } -#endif #if defined(HAS_SCALEROWDOWN2_16_HVX) if (TestCpuFlag(kCpuHasHVX)) { ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_16_HVX @@ -360,18 +365,6 @@ static void ScalePlaneDown4_16(int src_width, src_ptr += src_stride * 2; // Point to row 2. src_stride = 0; } -#if defined(HAS_SCALEROWDOWN4_16_NEON) - if (TestCpuFlag(kCpuHasNEON) && IS_ALIGNED(dst_width, 8)) { - ScaleRowDown4 = - filtering ? ScaleRowDown4Box_16_NEON : ScaleRowDown4_16_NEON; - } -#endif -#if defined(HAS_SCALEROWDOWN4_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleRowDown4 = - filtering ? ScaleRowDown4Box_16_SSE2 : ScaleRowDown4_16_SSE2; - } -#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -455,23 +448,23 @@ static void ScalePlaneDown34(int src_width, } } #endif -#if defined(HAS_SCALEROWDOWN34_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { +#if defined(HAS_SCALEROWDOWN34_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { if (dst_width % 24 == 0) { if (!filtering) { - ScaleRowDown34_0 = ScaleRowDown34_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_SSSE3; + ScaleRowDown34_0 = ScaleRowDown34_AVX2; + ScaleRowDown34_1 = ScaleRowDown34_AVX2; } else { - ScaleRowDown34_0 = ScaleRowDown34_0_Box_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_1_Box_SSSE3; + ScaleRowDown34_0 = ScaleRowDown34_0_Box_AVX2; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_AVX2; } } else { if (!filtering) { - ScaleRowDown34_0 = ScaleRowDown34_Any_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_Any_SSSE3; + ScaleRowDown34_0 = ScaleRowDown34_Any_AVX2; + ScaleRowDown34_1 = ScaleRowDown34_Any_AVX2; } else { - ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_SSSE3; + ScaleRowDown34_0 = ScaleRowDown34_0_Box_Any_AVX2; + ScaleRowDown34_1 = ScaleRowDown34_1_Box_Any_AVX2; } } } @@ -557,28 +550,6 @@ static void ScalePlaneDown34_16(int src_width, ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_C; ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_C; } -#if defined(HAS_SCALEROWDOWN34_16_NEON) - if (TestCpuFlag(kCpuHasNEON) && (dst_width % 24 == 0)) { - if (!filtering) { - ScaleRowDown34_0 = ScaleRowDown34_16_NEON; - ScaleRowDown34_1 = ScaleRowDown34_16_NEON; - } else { - ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_NEON; - ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_NEON; - } - } -#endif -#if defined(HAS_SCALEROWDOWN34_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) { - if (!filtering) { - ScaleRowDown34_0 = ScaleRowDown34_16_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_16_SSSE3; - } else { - ScaleRowDown34_0 = ScaleRowDown34_0_Box_16_SSSE3; - ScaleRowDown34_1 = ScaleRowDown34_1_Box_16_SSSE3; - } - } -#endif for (y = 0; y < dst_height - 2; y += 3) { ScaleRowDown34_0(src_ptr, filter_stride, dst_ptr, dst_width); @@ -664,22 +635,23 @@ static void ScalePlaneDown38(int src_width, } } #endif -#if defined(HAS_SCALEROWDOWN38_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { +#if defined(HAS_SCALEROWDOWN38_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { if (!filtering) { - ScaleRowDown38_3 = ScaleRowDown38_Any_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_Any_SSSE3; + ScaleRowDown38_3 = ScaleRowDown38_Any_AVX2; + ScaleRowDown38_2 = ScaleRowDown38_Any_AVX2; } else { - ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_SSSE3; + ScaleRowDown38_3 = ScaleRowDown38_3_Box_Any_AVX2; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_Any_AVX2; } - if (dst_width % 12 == 0 && !filtering) { - ScaleRowDown38_3 = ScaleRowDown38_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_SSSE3; - } - if (dst_width % 6 == 0 && filtering) { - ScaleRowDown38_3 = ScaleRowDown38_3_Box_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_2_Box_SSSE3; + if (dst_width % 12 == 0) { + if (!filtering) { + ScaleRowDown38_3 = ScaleRowDown38_AVX2; + ScaleRowDown38_2 = ScaleRowDown38_AVX2; + } else { + ScaleRowDown38_3 = ScaleRowDown38_3_Box_AVX2; + ScaleRowDown38_2 = ScaleRowDown38_2_Box_AVX2; + } } } #endif @@ -783,28 +755,6 @@ static void ScalePlaneDown38_16(int src_width, ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_C; ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_C; } -#if defined(HAS_SCALEROWDOWN38_16_NEON) - if (TestCpuFlag(kCpuHasNEON) && (dst_width % 12 == 0)) { - if (!filtering) { - ScaleRowDown38_3 = ScaleRowDown38_16_NEON; - ScaleRowDown38_2 = ScaleRowDown38_16_NEON; - } else { - ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_NEON; - ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_NEON; - } - } -#endif -#if defined(HAS_SCALEROWDOWN38_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && (dst_width % 24 == 0)) { - if (!filtering) { - ScaleRowDown38_3 = ScaleRowDown38_16_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_16_SSSE3; - } else { - ScaleRowDown38_3 = ScaleRowDown38_3_Box_16_SSSE3; - ScaleRowDown38_2 = ScaleRowDown38_2_Box_16_SSSE3; - } - } -#endif for (y = 0; y < dst_height - 2; y += 3) { ScaleRowDown38_3(src_ptr, filter_stride, dst_ptr, dst_width); @@ -1002,11 +952,15 @@ static int ScalePlaneBox(int src_width, ScaleAddRow = ScaleAddRow_AVX512BW; } #endif -#if defined(HAS_SCALEADDCOLS_AVX512BW) - if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0) { - ScaleAddCols = (dx & 0xffff) ? ScaleAddCols2_AVX512BW - : ((dx != 0x10000) ? ScaleAddCols1_AVX512BW - : ScaleAddCols0_C); +#if defined(HAS_SCALEADDCOLS1_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0 && !(dx & 0xffff) && + dx != 0x10000) { + ScaleAddCols = ScaleAddCols1_AVX512BW; + } +#endif +#if defined(HAS_SCALEADDCOLS2_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW) && dx > 0 && (dx & 0xffff)) { + ScaleAddCols = ScaleAddCols2_AVX512BW; } #endif #if defined(HAS_SCALEADDROW_NEON) @@ -1096,12 +1050,6 @@ static int ScalePlaneBox_16(int src_width, void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr, int src_width) = ScaleAddRow_16_C; -#if defined(HAS_SCALEADDROW_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(src_width, 16)) { - ScaleAddRow = ScaleAddRow_16_SSE2; - } -#endif - uint32_t uy = y; for (j = 0; j < dst_height; ++j) { int boxheight; @@ -1224,9 +1172,9 @@ static int ScalePlaneBilinearDown(int src_width, } #endif -#if defined(HAS_SCALEFILTERCOLS_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleFilterCols = ScaleFilterCols_SSSE3; +#if defined(HAS_SCALEFILTERCOLS_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_AVX2; } #endif #if defined(HAS_SCALEFILTERCOLS_NEON) @@ -1307,14 +1255,6 @@ static int ScalePlaneBilinearDown_16(int src_width, &dx, &dy); src_width = Abs(src_width); -#if defined(HAS_INTERPOLATEROW_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - InterpolateRow = InterpolateRow_16_Any_SSSE3; - if (IS_ALIGNED(src_width, 16)) { - InterpolateRow = InterpolateRow_16_SSSE3; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; @@ -1355,12 +1295,6 @@ static int ScalePlaneBilinearDown_16(int src_width, } #endif -#if defined(HAS_SCALEFILTERCOLS_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleFilterCols = ScaleFilterCols_16_SSSE3; - } -#endif - uint32_t uy = y; for (j = 0; j < dst_height; ++j) { if (uy > max_y) { @@ -1468,9 +1402,9 @@ static int ScalePlaneBilinearUp(int src_width, if (filtering && src_width >= 32768) { ScaleFilterCols = ScaleFilterCols64_C; } -#if defined(HAS_SCALEFILTERCOLS_SSSE3) - if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleFilterCols = ScaleFilterCols_SSSE3; +#if defined(HAS_SCALEFILTERCOLS_AVX2) + if (filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleFilterCols = ScaleFilterCols_AVX2; } #endif #if defined(HAS_SCALEFILTERCOLS_NEON) @@ -1499,9 +1433,9 @@ static int ScalePlaneBilinearUp(int src_width, #endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleFilterCols = ScaleColsUp2_C; -#if defined(HAS_SCALECOLSUP2_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleFilterCols = ScaleColsUp2_SSE2; +#if defined(HAS_SCALECOLSUP2_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) { + ScaleFilterCols = ScaleColsUp2_AVX2; } #endif } @@ -1937,14 +1871,6 @@ static int ScalePlaneBilinearUp_16(int src_width, assert(dy <= 65536); src_width = Abs(src_width); -#if defined(HAS_INTERPOLATEROW_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - InterpolateRow = InterpolateRow_16_Any_SSSE3; - if (IS_ALIGNED(dst_width, 16)) { - InterpolateRow = InterpolateRow_16_SSSE3; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; @@ -1988,18 +1914,8 @@ static int ScalePlaneBilinearUp_16(int src_width, if (filtering && src_width >= 32768) { ScaleFilterCols = ScaleFilterCols64_16_C; } -#if defined(HAS_SCALEFILTERCOLS_16_SSSE3) - if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleFilterCols = ScaleFilterCols_16_SSSE3; - } -#endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleFilterCols = ScaleColsUp2_16_C; -#if defined(HAS_SCALECOLS_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleFilterCols = ScaleColsUp2_16_SSE2; - } -#endif } uint32_t uy = y; @@ -2093,9 +2009,9 @@ static void ScalePlaneSimple(int src_width, if (src_width * 2 == dst_width && x < 0x8000) { ScaleCols = ScaleColsUp2_C; -#if defined(HAS_SCALECOLSUP2_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleCols = ScaleColsUp2_SSE2; +#if defined(HAS_SCALECOLSUP2_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) { + ScaleCols = ScaleColsUp2_AVX2; } #endif } @@ -2131,11 +2047,6 @@ static void ScalePlaneSimple_16(int src_width, if (src_width * 2 == dst_width && x < 0x8000) { ScaleCols = ScaleColsUp2_16_C; -#if defined(HAS_SCALECOLS_16_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleCols = ScaleColsUp2_16_SSE2; - } -#endif } uint32_t uy = y; diff --git a/source/scale_any.cc b/source/scale_any.cc index 713f3f186..ec903768e 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -50,6 +50,19 @@ extern "C" { dst_ptr + np * BPP, r + 1); \ } +// Fixed scale down for 16-bit planar. +#define SDANY16(NAMEANY, SCALEROWDOWN_SIMD, SCALEROWDOWN_C, FACTOR, MASK) \ + void NAMEANY(const uint16_t* src_ptr, ptrdiff_t src_stride, \ + uint16_t* dst_ptr, int dst_width) { \ + int r = (int)((unsigned int)dst_width % (MASK + 1)); /* NOLINT */ \ + int n = dst_width - r; \ + if (n > 0) { \ + SCALEROWDOWN_SIMD(src_ptr, src_stride, dst_ptr, n); \ + } \ + ptrdiff_t np = n; \ + SCALEROWDOWN_C(src_ptr + (np * FACTOR), src_stride, dst_ptr + np, r); \ + } + #ifdef HAS_SCALEROWDOWN2_SSSE3 SDANY(ScaleRowDown2_Any_SSSE3, ScaleRowDown2_SSSE3, ScaleRowDown2_C, 2, 1, 15) SDANY(ScaleRowDown2Linear_Any_SSSE3, @@ -71,13 +84,21 @@ SDODD(ScaleRowDown2Box_Odd_SSSE3, 1, 15) #endif -#ifdef HAS_SCALEUVROWDOWN2BOX_SSSE3 -SDANY(ScaleUVRowDown2Box_Any_SSSE3, - ScaleUVRowDown2Box_SSSE3, - ScaleUVRowDown2Box_C, +#ifdef HAS_SCALEUVROWDOWN2_AVX2 +SDANY(ScaleUVRowDown2_Any_AVX2, + ScaleUVRowDown2_AVX2, + ScaleUVRowDown2_C, 2, 2, - 3) + 7) +#endif +#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2 +SDANY(ScaleUVRowDown2Linear_Any_AVX2, + ScaleUVRowDown2Linear_AVX2, + ScaleUVRowDown2Linear_C, + 2, + 2, + 7) #endif #ifdef HAS_SCALEUVROWDOWN2BOX_AVX2 SDANY(ScaleUVRowDown2Box_Any_AVX2, @@ -108,6 +129,23 @@ SDODD(ScaleRowDown2Box_Odd_AVX2, 1, 31) #endif +#ifdef HAS_SCALEROWDOWN2_16_AVX2 +SDANY16(ScaleRowDown2_16_Any_AVX2, + ScaleRowDown2_16_AVX2, + ScaleRowDown2_16_C, + 2, + 15) +SDANY16(ScaleRowDown2Linear_16_Any_AVX2, + ScaleRowDown2Linear_16_AVX2, + ScaleRowDown2Linear_16_C, + 2, + 15) +SDANY16(ScaleRowDown2Box_16_Any_AVX2, + ScaleRowDown2Box_16_AVX2, + ScaleRowDown2Box_16_C, + 2, + 15) +#endif #ifdef HAS_SCALEROWDOWN2_NEON SDANY(ScaleRowDown2_Any_NEON, ScaleRowDown2_NEON, ScaleRowDown2_C, 2, 1, 15) SDANY(ScaleRowDown2Linear_Any_NEON, @@ -204,21 +242,21 @@ SDANY(ScaleRowDown4Box_Any_LSX, 1, 7) #endif -#ifdef HAS_SCALEROWDOWN34_SSSE3 -SDANY(ScaleRowDown34_Any_SSSE3, - ScaleRowDown34_SSSE3, +#ifdef HAS_SCALEROWDOWN34_AVX2 +SDANY(ScaleRowDown34_Any_AVX2, + ScaleRowDown34_AVX2, ScaleRowDown34_C, 4 / 3, 1, 23) -SDANY(ScaleRowDown34_0_Box_Any_SSSE3, - ScaleRowDown34_0_Box_SSSE3, +SDANY(ScaleRowDown34_0_Box_Any_AVX2, + ScaleRowDown34_0_Box_AVX2, ScaleRowDown34_0_Box_C, 4 / 3, 1, 23) -SDANY(ScaleRowDown34_1_Box_Any_SSSE3, - ScaleRowDown34_1_Box_SSSE3, +SDANY(ScaleRowDown34_1_Box_Any_AVX2, + ScaleRowDown34_1_Box_AVX2, ScaleRowDown34_1_Box_C, 4 / 3, 1, @@ -285,25 +323,25 @@ SDANY(ScaleRowDown34_1_Box_Any_LSX, 1, 11) #endif -#ifdef HAS_SCALEROWDOWN38_SSSE3 -SDANY(ScaleRowDown38_Any_SSSE3, - ScaleRowDown38_SSSE3, +#ifdef HAS_SCALEROWDOWN38_AVX2 +SDANY(ScaleRowDown38_Any_AVX2, + ScaleRowDown38_AVX2, ScaleRowDown38_C, 8 / 3, 1, 11) -SDANY(ScaleRowDown38_3_Box_Any_SSSE3, - ScaleRowDown38_3_Box_SSSE3, +SDANY(ScaleRowDown38_3_Box_Any_AVX2, + ScaleRowDown38_3_Box_AVX2, ScaleRowDown38_3_Box_C, 8 / 3, 1, - 5) -SDANY(ScaleRowDown38_2_Box_Any_SSSE3, - ScaleRowDown38_2_Box_SSSE3, + 11) +SDANY(ScaleRowDown38_2_Box_Any_AVX2, + ScaleRowDown38_2_Box_AVX2, ScaleRowDown38_2_Box_C, 8 / 3, 1, - 5) + 11) #endif #ifdef HAS_SCALEROWDOWN38_NEON SDANY(ScaleRowDown38_Any_NEON, @@ -346,25 +384,25 @@ SDANY(ScaleRowDown38_2_Box_Any_LSX, 11) #endif -#ifdef HAS_SCALEARGBROWDOWN2_SSE2 -SDANY(ScaleARGBRowDown2_Any_SSE2, - ScaleARGBRowDown2_SSE2, +#ifdef HAS_SCALEARGBROWDOWN2_AVX2 +SDANY(ScaleARGBRowDown2_Any_AVX2, + ScaleARGBRowDown2_AVX2, ScaleARGBRowDown2_C, 2, 4, - 3) -SDANY(ScaleARGBRowDown2Linear_Any_SSE2, - ScaleARGBRowDown2Linear_SSE2, + 7) +SDANY(ScaleARGBRowDown2Linear_Any_AVX2, + ScaleARGBRowDown2Linear_AVX2, ScaleARGBRowDown2Linear_C, 2, 4, - 3) -SDANY(ScaleARGBRowDown2Box_Any_SSE2, - ScaleARGBRowDown2Box_SSE2, + 7) +SDANY(ScaleARGBRowDown2Box_Any_AVX2, + ScaleARGBRowDown2Box_AVX2, ScaleARGBRowDown2Box_C, 2, 4, - 3) + 7) #endif #ifdef HAS_SCALEARGBROWDOWN2_NEON SDANY(ScaleARGBRowDown2_Any_NEON, @@ -506,6 +544,8 @@ SDANY(ScaleARGBRowDown2Box_Any_WASMSIMD, 4, 3) #endif +#undef SDANY16 +#undef SDODD #undef SDANY // Scale down by even scale factor. @@ -522,17 +562,17 @@ SDANY(ScaleARGBRowDown2Box_Any_WASMSIMD, dst_ptr + np * BPP, r); \ } -#ifdef HAS_SCALEARGBROWDOWNEVEN_SSE2 -SDAANY(ScaleARGBRowDownEven_Any_SSE2, - ScaleARGBRowDownEven_SSE2, +#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2 +SDAANY(ScaleARGBRowDownEven_Any_AVX2, + ScaleARGBRowDownEven_AVX2, ScaleARGBRowDownEven_C, 4, - 3) -SDAANY(ScaleARGBRowDownEvenBox_Any_SSE2, - ScaleARGBRowDownEvenBox_SSE2, + 7) +SDAANY(ScaleARGBRowDownEvenBox_Any_AVX2, + ScaleARGBRowDownEvenBox_AVX2, ScaleARGBRowDownEvenBox_C, 4, - 3) + 7) #endif #ifdef HAS_SCALEARGBROWDOWNEVEN_NEON SDAANY(ScaleARGBRowDownEven_Any_NEON, @@ -558,6 +598,13 @@ SDAANY(ScaleARGBRowDownEvenBox_Any_LSX, 4, 3) #endif +#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2 +SDAANY(ScaleUVRowDownEven_Any_AVX2, + ScaleUVRowDownEven_AVX2, + ScaleUVRowDownEven_C, + 2, + 7) +#endif #ifdef HAS_SCALEUVROWDOWNEVEN_NEON SDAANY(ScaleUVRowDownEven_Any_NEON, ScaleUVRowDownEven_NEON, @@ -1107,7 +1154,7 @@ SBU2BLANY(ScaleUVRowUp2_Bilinear_Any_AVX2, SBU2BLANY(ScaleUVRowUp2_Bilinear_16_Any_SSE41, ScaleUVRowUp2_Bilinear_16_SSE41, ScaleUVRowUp2_Bilinear_16_C, - 7, + 3, uint16_t) #endif diff --git a/source/scale_argb.cc b/source/scale_argb.cc index dd9df3612..53280ea16 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -67,19 +67,19 @@ static void ScaleARGBDown2(int src_width, src_argb += (y >> 16) * src_stride + ((x >> 16) - 1) * 4; } -#if defined(HAS_SCALEARGBROWDOWN2_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { +#if defined(HAS_SCALEARGBROWDOWN2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { ScaleARGBRowDown2 = filtering == kFilterNone - ? ScaleARGBRowDown2_Any_SSE2 - : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_SSE2 - : ScaleARGBRowDown2Box_Any_SSE2); - if (IS_ALIGNED(dst_width, 4)) { + ? ScaleARGBRowDown2_Any_AVX2 + : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_Any_AVX2 + : ScaleARGBRowDown2Box_Any_AVX2); + if (IS_ALIGNED(dst_width, 8)) { ScaleARGBRowDown2 = filtering == kFilterNone - ? ScaleARGBRowDown2_SSE2 - : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_SSE2 - : ScaleARGBRowDown2Box_SSE2); + ? ScaleARGBRowDown2_AVX2 + : (filtering == kFilterLinear ? ScaleARGBRowDown2Linear_AVX2 + : ScaleARGBRowDown2Box_AVX2); } } #endif @@ -205,11 +205,11 @@ static int ScaleARGBDown4Box(int src_width, (void)dx; assert(dx == 65536 * 4); // Test scale factor of 4. assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4. -#if defined(HAS_SCALEARGBROWDOWN2_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_SSE2; - if (IS_ALIGNED(dst_width, 4)) { - ScaleARGBRowDown2 = ScaleARGBRowDown2Box_SSE2; +#if defined(HAS_SCALEARGBROWDOWN2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + ScaleARGBRowDown2 = ScaleARGBRowDown2Box_AVX2; } } #endif @@ -284,13 +284,13 @@ static void ScaleARGBDownEven(int src_width, assert(IS_ALIGNED(src_width, 2)); assert(IS_ALIGNED(src_height, 2)); src_argb += (y >> 16) * src_stride + (x >> 16) * 4; -#if defined(HAS_SCALEARGBROWDOWNEVEN_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_SSE2 - : ScaleARGBRowDownEven_Any_SSE2; - if (IS_ALIGNED(dst_width, 4)) { +#if defined(HAS_SCALEARGBROWDOWNEVEN_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleARGBRowDownEven = filtering ? ScaleARGBRowDownEvenBox_Any_AVX2 + : ScaleARGBRowDownEven_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { ScaleARGBRowDownEven = - filtering ? ScaleARGBRowDownEvenBox_SSE2 : ScaleARGBRowDownEven_SSE2; + filtering ? ScaleARGBRowDownEvenBox_AVX2 : ScaleARGBRowDownEven_AVX2; } } #endif @@ -444,9 +444,9 @@ static int ScaleARGBBilinearDown(int src_width, } } #endif -#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3; +#if defined(HAS_SCALEARGBFILTERCOLS_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_AVX2; } #endif #if defined(HAS_SCALEARGBFILTERCOLS_NEON) @@ -607,9 +607,9 @@ static int ScaleARGBBilinearUp(int src_width, ScaleARGBFilterCols = filtering ? ScaleARGBFilterCols64_C : ScaleARGBCols64_C; } -#if defined(HAS_SCALEARGBFILTERCOLS_SSSE3) - if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleARGBFilterCols = ScaleARGBFilterCols_SSSE3; +#if defined(HAS_SCALEARGBFILTERCOLS_AVX2) + if (filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBFilterCols_AVX2; } #endif #if defined(HAS_SCALEARGBFILTERCOLS_NEON) @@ -633,9 +633,9 @@ static int ScaleARGBBilinearUp(int src_width, ScaleARGBFilterCols = ScaleARGBFilterCols_RVV; } #endif -#if defined(HAS_SCALEARGBCOLS_SSE2) - if (!filtering && TestCpuFlag(kCpuHasSSE2) && src_width < 32768) { - ScaleARGBFilterCols = ScaleARGBCols_SSE2; +#if defined(HAS_SCALEARGBCOLS_AVX2) + if (!filtering && TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleARGBFilterCols = ScaleARGBCols_AVX2; } #endif #if defined(HAS_SCALEARGBCOLS_NEON) @@ -672,9 +672,9 @@ static int ScaleARGBBilinearUp(int src_width, #endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleARGBFilterCols = ScaleARGBColsUp2_C; -#if defined(HAS_SCALEARGBCOLSUP2_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleARGBFilterCols = ScaleARGBColsUp2_SSE2; +#if defined(HAS_SCALEARGBCOLSUP2_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBFilterCols = ScaleARGBColsUp2_AVX2; } #endif #if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD) @@ -772,9 +772,9 @@ static void ScaleARGBSimple(int src_width, int dst_width, int x, int dx) = (src_width >= 32768) ? ScaleARGBCols64_C : ScaleARGBCols_C; const int64_t max_y = (int64_t)(src_height - 1) << 16; -#if defined(HAS_SCALEARGBCOLS_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && src_width < 32768) { - ScaleARGBCols = ScaleARGBCols_SSE2; +#if defined(HAS_SCALEARGBCOLS_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && src_width < 32768) { + ScaleARGBCols = ScaleARGBCols_AVX2; } #endif #if defined(HAS_SCALEARGBCOLS_NEON) @@ -803,9 +803,9 @@ static void ScaleARGBSimple(int src_width, #endif if (src_width * 2 == dst_width && x < 0x8000) { ScaleARGBCols = ScaleARGBColsUp2_C; -#if defined(HAS_SCALEARGBCOLSUP2_SSE2) - if (TestCpuFlag(kCpuHasSSE2) && IS_ALIGNED(dst_width, 8)) { - ScaleARGBCols = ScaleARGBColsUp2_SSE2; +#if defined(HAS_SCALEARGBCOLSUP2_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && IS_ALIGNED(dst_width, 8)) { + ScaleARGBCols = ScaleARGBColsUp2_AVX2; } #endif #if defined(HAS_SCALEARGBCOLSUP2_WASMSIMD) diff --git a/source/scale_common.cc b/source/scale_common.cc index 0f1e7c845..a07a2183e 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1155,6 +1155,86 @@ void ScaleUVRowDownEvenBox_C(const uint8_t* src_uv, } } +void ScaleUVRowDown2_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width) { + int x; + (void)src_stride; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = src_uv[2]; // Store the 2nd UV + dst_uv[1] = src_uv[3]; + src_uv += 4; + dst_uv += 2; + } +} + +void ScaleUVRowDown2Linear_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width) { + int x; + (void)src_stride; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] + 1) >> 1); + dst_uv[1] = (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + 1) >> 1); + src_uv += 4; + dst_uv += 2; + } +} + +void ScaleUVRowDown2Box_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + uint16_t* dst_uv, + int dst_width) { + int x; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] + + src_uv[src_stride] + src_uv[src_stride + 2] + 2) >> + 2); + dst_uv[1] = + (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + src_uv[src_stride + 1] + + src_uv[src_stride + 3] + 2) >> + 2); + src_uv += 4; + dst_uv += 2; + } +} + +void ScaleUVRowDownEven_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint16_t* dst_uv, + int dst_width) { + int x; + (void)src_stride; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = src_uv[0]; + dst_uv[1] = src_uv[1]; + src_uv += src_stepx * 2; + dst_uv += 2; + } +} + +void ScaleUVRowDownEvenBox_16_C(const uint16_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint16_t* dst_uv, + int dst_width) { + int x; + for (x = 0; x < dst_width; ++x) { + dst_uv[0] = (uint16_t)(((uint32_t)src_uv[0] + src_uv[2] + + src_uv[src_stride] + src_uv[src_stride + 2] + 2) >> + 2); + dst_uv[1] = + (uint16_t)(((uint32_t)src_uv[1] + src_uv[3] + src_uv[src_stride + 1] + + src_uv[src_stride + 3] + 2) >> + 2); + src_uv += src_stepx * 2; + dst_uv += 2; + } +} + void ScaleUVRowUp2_Linear_C(const uint8_t* src_ptr, uint8_t* dst_ptr, int dst_width) { @@ -1382,6 +1462,118 @@ void ScaleUVAddCols3_C(int dst_width, } } +// Scales a single row of 16-bit UV pixels using point sampling. +void ScaleUVCols_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx) { + int j; + for (j = 0; j < dst_width; ++j) { + const uint16_t* src = src_uv + (x >> 16) * 2; + dst_uv[0] = src[0]; + dst_uv[1] = src[1]; + dst_uv += 2; + x += dx; + } +} + +void ScaleUVCols64_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x32, + int dx) { + int64_t x = (int64_t)(x32); + int j; + for (j = 0; j < dst_width; ++j) { + const uint16_t* src = src_uv + (x >> 16) * 2; + dst_uv[0] = src[0]; + dst_uv[1] = src[1]; + dst_uv += 2; + x += dx; + } +} + +// Scales a single row of 16-bit UV pixels up by 2x using point sampling. +void ScaleUVColsUp2_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx) { + int j; + (void)x; + (void)dx; + for (j = 0; j < dst_width - 1; j += 2) { + dst_uv[0] = dst_uv[2] = src_uv[0]; + dst_uv[1] = dst_uv[3] = src_uv[1]; + src_uv += 2; + dst_uv += 4; + } + if (dst_width & 1) { + dst_uv[0] = src_uv[0]; + dst_uv[1] = src_uv[1]; + } +} + +#define BLENDER_16(a, b, f) \ + (uint16_t)((int)(a) + \ + (int)((((int64_t)((f)) * ((int64_t)(b) - (int)(a))) + 0x8000) >> \ + 16)) + +void ScaleUVFilterCols_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x, + int dx) { + int j; + for (j = 0; j < dst_width; ++j) { + int xi = (x >> 16) * 2; + int xf = x & 0xffff; + const uint16_t* a = src_uv + xi; + const uint16_t* b = a + 2; + dst_uv[0] = BLENDER_16(a[0], b[0], xf); + dst_uv[1] = BLENDER_16(a[1], b[1], xf); + dst_uv += 2; + x += dx; + } +} + +void ScaleUVFilterCols64_16_C(uint16_t* dst_uv, + const uint16_t* src_uv, + int dst_width, + int x32, + int dx) { + int64_t x = (int64_t)(x32); + int j; + for (j = 0; j < dst_width; ++j) { + int64_t xi = (x >> 16) * 2; + int xf = x & 0xffff; + const uint16_t* a = src_uv + xi; + const uint16_t* b = a + 2; + dst_uv[0] = BLENDER_16(a[0], b[0], xf); + dst_uv[1] = BLENDER_16(a[1], b[1], xf); + dst_uv += 2; + x += dx; + } +} +#undef BLENDER_16 + +// Scale 16-bit UV columns by 3 box filter. +void ScaleUVAddCols3_16_C(int dst_width, + int scaleval, + const uint32_t* src_uv, + uint16_t* dst_uv) { + int i; + for (i = 0; i < dst_width; ++i) { + uint32_t u = src_uv[0] + src_uv[2] + src_uv[4]; + uint32_t v = src_uv[1] + src_uv[3] + src_uv[5]; + src_uv += 6; + dst_uv[0] = (uint16_t)((u * (uint32_t)scaleval) >> 16); + dst_uv[1] = (uint16_t)((v * (uint32_t)scaleval) >> 16); + dst_uv += 2; + } +} + // Scale plane vertically with bilinear interpolation. void ScalePlaneVertical(int src_height, int dst_width, @@ -1536,14 +1728,6 @@ void ScalePlaneVertical_16(int src_height, assert(dst_width > 0); assert(dst_height > 0); src_argb += (x >> 16) * wpp; -#if defined(HAS_INTERPOLATEROW_16_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - InterpolateRow = InterpolateRow_16_Any_SSSE3; - if (IS_ALIGNED(dst_width_words, 16)) { - InterpolateRow = InterpolateRow_16_SSSE3; - } - } -#endif #if defined(HAS_INTERPOLATEROW_16_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { InterpolateRow = InterpolateRow_16_Any_AVX2; diff --git a/source/scale_gcc.cc b/source/scale_gcc.cc index b018af795..dc0383c37 100644 --- a/source/scale_gcc.cc +++ b/source/scale_gcc.cc @@ -28,6 +28,7 @@ extern "C" { (defined(__x86_64__) || defined(__i386__)) && \ !defined(LIBYUV_ENABLE_ROWWIN) +#ifdef HAS_SCALEROWDOWN34_AVX2 // Offsets for source bytes 0 to 9 static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9, 128, 128, 128, 128, 128, 128, 128, 128}; @@ -62,7 +63,9 @@ static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; // Coefficients for source bytes 21 to 31 static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2}; +#endif // HAS_SCALEROWDOWN34_AVX2 +#ifdef HAS_SCALEROWDOWN38_AVX2 static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; @@ -96,6 +99,7 @@ static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128, // Scaling values for boxes of 3x2 and 2x2 static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, 65536 / 3, 65536 / 2, 0, 0}; +#endif // HAS_SCALEROWDOWN38_AVX2 // GCC versions of row functions are verbatim conversions from Visual C. // Generated using gcc disassembly on Visual C object file: @@ -298,6 +302,118 @@ void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, } #endif // HAS_SCALEROWDOWN2_AVX2 +#ifdef HAS_SCALEROWDOWN2_16_AVX2 +void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + (void)src_stride; + asm volatile( + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm0 \n" + "vpsrld $0x10,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "xmm0", "xmm1"); +} + +void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + (void)src_stride; + asm volatile( + "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n" + "vpsrld $0x10,%%ymm4,%%ymm4 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm2 \n" + "vpsrld $0x10,%%ymm1,%%ymm3 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpand %%ymm4,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpackusdw %%ymm3,%%ymm2,%%ymm2 \n" + "vpavgw %%ymm2,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); +} + +void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + asm volatile( + "vpcmpeqd %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrld $0x1f,%%ymm6,%%ymm7 \n" + "vpslld $0x1,%%ymm7,%%ymm7 \n" + "vpsrld $0x10,%%ymm6,%%ymm6 \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu 0x00(%0,%3,2),%%ymm2 \n" + "vmovdqu 0x20(%0,%3,2),%%ymm3 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm4 \n" + "vpand %%ymm6,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm4,%%ymm0,%%ymm0 \n" + "vpsrld $0x10,%%ymm2,%%ymm4 \n" + "vpand %%ymm6,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm4,%%ymm2,%%ymm2 \n" + "vpaddd %%ymm2,%%ymm0,%%ymm0 \n" + "vpsrld $0x10,%%ymm1,%%ymm4 \n" + "vpand %%ymm6,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm4,%%ymm1,%%ymm1 \n" + "vpsrld $0x10,%%ymm3,%%ymm4 \n" + "vpand %%ymm6,%%ymm3,%%ymm3 \n" + "vpaddd %%ymm4,%%ymm3,%%ymm3 \n" + "vpaddd %%ymm3,%%ymm1,%%ymm1 \n" + "vpaddd %%ymm7,%%ymm0,%%ymm0 \n" + "vpaddd %%ymm7,%%ymm1,%%ymm1 \n" + "vpsrld $0x2,%%ymm0,%%ymm0 \n" + "vpsrld $0x2,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jg 1b \n" + "vzeroupper \n" + : "+r"(src_ptr), // %0 + "+r"(dst_ptr), // %1 + "+r"(dst_width) // %2 + : "r"(src_stride) // %3 + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm6", "xmm7"); +} +#endif // HAS_SCALEROWDOWN2_16_AVX2 + #ifdef HAS_SCALEROWDOWN2_AVX512BW void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -624,266 +740,418 @@ void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, } #endif // HAS_SCALEROWDOWN4_AVX2 -void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +#ifdef HAS_SCALEROWDOWN34_AVX2 +void ScaleRowDown34_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { (void)src_stride; asm volatile( - "movdqa %[kShuf0],%%xmm3 \n" - "movdqa %[kShuf1],%%xmm4 \n" - "movdqa %[kShuf2],%%xmm5 \n" - "1: \n" - "movdqu (%[src_ptr]),%%xmm0 \n" - "movdqu 0x10(%[src_ptr]),%%xmm2 \n" - "lea 0x20(%[src_ptr]),%[src_ptr] \n" - "movdqa %%xmm2,%%xmm1 \n" - "palignr $0x8,%%xmm0,%%xmm1 \n" - "pshufb %%xmm3,%%xmm0 \n" - "pshufb %%xmm4,%%xmm1 \n" - "pshufb %%xmm5,%%xmm2 \n" - "movq %%xmm0,(%[dst_ptr]) \n" - "movq %%xmm1,0x8(%[dst_ptr]) \n" - "movq %%xmm2,0x10(%[dst_ptr]) \n" - "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x18,%[dst_width] \n" - "jg 1b \n" - : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) - : [kShuf0] "m"(kShuf0), [kShuf1] "m"(kShuf1), [kShuf2] "m"(kShuf2) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); -} - -void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - asm volatile( - "movdqa %[kShuf01],%%xmm2 \n" - "movdqa %[kShuf11],%%xmm3 \n" - "movdqa %[kShuf21],%%xmm4 \n" - "movdqa %[kMadd01],%%xmm5 \n" - "movdqa %[kMadd11],%%xmm0 \n" - "movdqa %[kRound34],%%xmm1 \n" - "1: \n" - "movdqu (%[src_ptr]),%%xmm6 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm2,%%xmm6 \n" - "pmaddubsw %%xmm5,%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,(%[dst_ptr]) \n" - "movdqu 0x8(%[src_ptr]),%%xmm6 \n" - "movdqu 0x8(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm3,%%xmm6 \n" - "pmaddubsw %%xmm0,%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,0x8(%[dst_ptr]) \n" - "movdqu 0x10(%[src_ptr]),%%xmm6 \n" - "movdqu 0x10(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "lea 0x20(%[src_ptr]),%[src_ptr] \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm4,%%xmm6 \n" - "pmaddubsw %[kMadd21],%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,0x10(%[dst_ptr]) \n" - "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x18,%[dst_width] \n" - "jg 1b \n" - : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) - : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), - [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), - [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); -} - -void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - asm volatile( - "movdqa %[kShuf01],%%xmm2 \n" // kShuf01 - "movdqa %[kShuf11],%%xmm3 \n" // kShuf11 - "movdqa %[kShuf21],%%xmm4 \n" // kShuf21 - "movdqa %[kMadd01],%%xmm5 \n" // kMadd01 - "movdqa %[kMadd11],%%xmm0 \n" // kMadd11 - "movdqa %[kRound34],%%xmm1 \n" // kRound34 - "1: \n" - "movdqu (%[src_ptr]),%%xmm6 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "pavgb %%xmm6,%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm2,%%xmm6 \n" - "pmaddubsw %%xmm5,%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,(%[dst_ptr]) \n" - "movdqu 0x8(%[src_ptr]),%%xmm6 \n" - "movdqu 0x8(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "pavgb %%xmm6,%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm3,%%xmm6 \n" - "pmaddubsw %%xmm0,%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,0x8(%[dst_ptr]) \n" - "movdqu 0x10(%[src_ptr]),%%xmm6 \n" - "movdqu 0x10(%[src_ptr],%[src_stride],1),%%xmm7 \n" - "lea 0x20(%[src_ptr]),%[src_ptr] \n" - "pavgb %%xmm6,%%xmm7 \n" - "pavgb %%xmm7,%%xmm6 \n" - "pshufb %%xmm4,%%xmm6 \n" - "pmaddubsw %[kMadd21],%%xmm6 \n" - "paddsw %%xmm1,%%xmm6 \n" - "psrlw $0x2,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movq %%xmm6,0x10(%[dst_ptr]) \n" - "lea 0x18(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x18,%[dst_width] \n" - "jg 1b \n" - : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) - : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), - [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), - [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); -} - -void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - (void)src_stride; - asm volatile( - "movdqa %3,%%xmm4 \n" - "movdqa %4,%%xmm5 \n" + "vbroadcasti128 %[kShuf0],%%ymm3 \n" + "vbroadcasti128 %[kShuf1],%%ymm4 \n" + "vbroadcasti128 %[kShuf2],%%ymm5 \n" + "sub $0x30,%[dst_width] \n" + "jl 2f \n" LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "lea 0x20(%0),%0 \n" - "pshufb %%xmm4,%%xmm0 \n" - "pshufb %%xmm5,%%xmm1 \n" - "paddusb %%xmm1,%%xmm0 \n" - "movq %%xmm0,(%1) \n" - "movhlps %%xmm0,%%xmm1 \n" - "movd %%xmm1,0x8(%1) \n" - "lea 0xc(%1),%1 \n" - "sub $0xc,%2 \n" - "jg 1b \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n" + "lea 0x40(%[src_ptr]),%[src_ptr] \n" + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n" + "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm6 \n" + "vpalignr $0x8,%%ymm2,%%ymm6,%%ymm1 \n" + "vpshufb %%ymm3,%%ymm2,%%ymm0 \n" + "vpshufb %%ymm4,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm5,%%ymm6,%%ymm2 \n" + "vpunpcklqdq %%ymm1,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%[dst_ptr]) \n" + "vmovq %%xmm2,0x10(%[dst_ptr]) \n" + "vextracti128 $0x1,%%ymm0,%%xmm0 \n" + "vextracti128 $0x1,%%ymm2,%%xmm2 \n" + "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n" + "vmovq %%xmm2,0x28(%[dst_ptr]) \n" + "lea 0x30(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x30,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x30,%[dst_width] \n" + "jle 99f \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm2 \n" + "vpalignr $0x8,%%xmm0,%%xmm2,%%xmm1 \n" + "vpshufb %%xmm3,%%xmm0,%%xmm0 \n" + "vpshufb %%xmm4,%%xmm1,%%xmm1 \n" + "vpshufb %%xmm5,%%xmm2,%%xmm2 \n" + "vpunpcklqdq %%xmm1,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst_ptr]) \n" + "vmovq %%xmm2,0x10(%[dst_ptr]) \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), + [dst_width] "+r"(dst_width) + : [kShuf0] "m"(kShuf0), [kShuf1] "m"(kShuf1), [kShuf2] "m"(kShuf2) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); +} + +void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + asm volatile( + "vbroadcasti128 %[kRound34],%%ymm5 \n" + "sub $0x30,%[dst_width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm2 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n" + "vmovdqu 0x20(%[src_ptr],%[src_stride],1),%%ymm3 \n" + "lea 0x40(%[src_ptr]),%[src_ptr] \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n" + "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm3 \n" + "vpalignr $0x8,%%ymm2,%%ymm3,%%ymm4 \n" + + "vbroadcasti128 %[kShuf01],%%ymm6 \n" + "vbroadcasti128 %[kMadd01],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n" + "vpaddsw %%ymm5,%%ymm2,%%ymm2 \n" + "vpsrlw $0x2,%%ymm2,%%ymm2 \n" + "vpackuswb %%ymm2,%%ymm2,%%ymm2 \n" + + "vbroadcasti128 %[kShuf11],%%ymm6 \n" + "vbroadcasti128 %[kMadd11],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm4,%%ymm4 \n" + "vpmaddubsw %%ymm7,%%ymm4,%%ymm4 \n" + "vpaddsw %%ymm5,%%ymm4,%%ymm4 \n" + "vpsrlw $0x2,%%ymm4,%%ymm4 \n" + "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n" + + "vbroadcasti128 %[kShuf21],%%ymm6 \n" + "vbroadcasti128 %[kMadd21],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm3 \n" + "vpmaddubsw %%ymm7,%%ymm3,%%ymm3 \n" + "vpaddsw %%ymm5,%%ymm3,%%ymm3 \n" + "vpsrlw $0x2,%%ymm3,%%ymm3 \n" + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" + + "vpunpcklqdq %%ymm4,%%ymm2,%%ymm2 \n" + "vmovdqu %%xmm2,(%[dst_ptr]) \n" + "vmovq %%xmm3,0x10(%[dst_ptr]) \n" + "vextracti128 $0x1,%%ymm2,%%xmm0 \n" + "vextracti128 $0x1,%%ymm3,%%xmm1 \n" + "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n" + "vmovq %%xmm1,0x28(%[dst_ptr]) \n" + "lea 0x30(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x30,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x30,%[dst_width] \n" + "jle 99f \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n" + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm3 \n" + "vpalignr $0x8,%%xmm0,%%xmm3,%%xmm4 \n" + + "vpshufb %[kShuf01],%%xmm0,%%xmm0 \n" + "vpmaddubsw %[kMadd01],%%xmm0,%%xmm0 \n" + "vpaddsw %%xmm5,%%xmm0,%%xmm0 \n" + "vpsrlw $0x2,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + + "vpshufb %[kShuf11],%%xmm4,%%xmm4 \n" + "vpmaddubsw %[kMadd11],%%xmm4,%%xmm4 \n" + "vpaddsw %%xmm5,%%xmm4,%%xmm4 \n" + "vpsrlw $0x2,%%xmm4,%%xmm4 \n" + "vpackuswb %%xmm4,%%xmm4,%%xmm4 \n" + + "vpshufb %[kShuf21],%%xmm3,%%xmm3 \n" + "vpmaddubsw %[kMadd21],%%xmm3,%%xmm3 \n" + "vpaddsw %%xmm5,%%xmm3,%%xmm3 \n" + "vpsrlw $0x2,%%xmm3,%%xmm3 \n" + "vpackuswb %%xmm3,%%xmm3,%%xmm3 \n" + + "vpunpcklqdq %%xmm4,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst_ptr]) \n" + "vmovq %%xmm3,0x10(%[dst_ptr]) \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), + [dst_width] "+rm"(dst_width) + : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), + [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), + [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} + +void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + asm volatile( + "vbroadcasti128 %[kRound34],%%ymm5 \n" + "sub $0x30,%[dst_width] \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm2 \n" + "vpavgb %%ymm0,%%ymm2,%%ymm2 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vmovdqu 0x20(%[src_ptr]),%%ymm1 \n" + "vmovdqu 0x20(%[src_ptr],%[src_stride],1),%%ymm3 \n" + "lea 0x40(%[src_ptr]),%[src_ptr] \n" + "vpavgb %%ymm1,%%ymm3,%%ymm3 \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vperm2i128 $0x20,%%ymm1,%%ymm0,%%ymm2 \n" + "vperm2i128 $0x31,%%ymm1,%%ymm0,%%ymm3 \n" + "vpalignr $0x8,%%ymm2,%%ymm3,%%ymm4 \n" + + "vbroadcasti128 %[kShuf01],%%ymm6 \n" + "vbroadcasti128 %[kMadd01],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm2,%%ymm2 \n" + "vpmaddubsw %%ymm7,%%ymm2,%%ymm2 \n" + "vpaddsw %%ymm5,%%ymm2,%%ymm2 \n" + "vpsrlw $0x2,%%ymm2,%%ymm2 \n" + "vpackuswb %%ymm2,%%ymm2,%%ymm2 \n" + + "vbroadcasti128 %[kShuf11],%%ymm6 \n" + "vbroadcasti128 %[kMadd11],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm4,%%ymm4 \n" + "vpmaddubsw %%ymm7,%%ymm4,%%ymm4 \n" + "vpaddsw %%ymm5,%%ymm4,%%ymm4 \n" + "vpsrlw $0x2,%%ymm4,%%ymm4 \n" + "vpackuswb %%ymm4,%%ymm4,%%ymm4 \n" + + "vbroadcasti128 %[kShuf21],%%ymm6 \n" + "vbroadcasti128 %[kMadd21],%%ymm7 \n" + "vpshufb %%ymm6,%%ymm3,%%ymm3 \n" + "vpmaddubsw %%ymm7,%%ymm3,%%ymm3 \n" + "vpaddsw %%ymm5,%%ymm3,%%ymm3 \n" + "vpsrlw $0x2,%%ymm3,%%ymm3 \n" + "vpackuswb %%ymm3,%%ymm3,%%ymm3 \n" + + "vpunpcklqdq %%ymm4,%%ymm2,%%ymm2 \n" + "vmovdqu %%xmm2,(%[dst_ptr]) \n" + "vmovq %%xmm3,0x10(%[dst_ptr]) \n" + "vextracti128 $0x1,%%ymm2,%%xmm0 \n" + "vextracti128 $0x1,%%ymm3,%%xmm1 \n" + "vmovdqu %%xmm0,0x18(%[dst_ptr]) \n" + "vmovq %%xmm1,0x28(%[dst_ptr]) \n" + "lea 0x30(%[dst_ptr]),%[dst_ptr] \n" + "sub $0x30,%[dst_width] \n" + "jge 1b \n" + + "2: \n" + "add $0x30,%[dst_width] \n" + "jle 99f \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n" + "vpavgb %%ymm0,%%ymm1,%%ymm1 \n" + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm3 \n" + "vpalignr $0x8,%%xmm0,%%xmm3,%%xmm4 \n" + + "vpshufb %[kShuf01],%%xmm0,%%xmm0 \n" + "vpmaddubsw %[kMadd01],%%xmm0,%%xmm0 \n" + "vpaddsw %%xmm5,%%xmm0,%%xmm0 \n" + "vpsrlw $0x2,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + + "vpshufb %[kShuf11],%%xmm4,%%xmm4 \n" + "vpmaddubsw %[kMadd11],%%xmm4,%%xmm4 \n" + "vpaddsw %%xmm5,%%xmm4,%%xmm4 \n" + "vpsrlw $0x2,%%xmm4,%%xmm4 \n" + "vpackuswb %%xmm4,%%xmm4,%%xmm4 \n" + + "vpshufb %[kShuf21],%%xmm3,%%xmm3 \n" + "vpmaddubsw %[kMadd21],%%xmm3,%%xmm3 \n" + "vpaddsw %%xmm5,%%xmm3,%%xmm3 \n" + "vpsrlw $0x2,%%xmm3,%%xmm3 \n" + "vpackuswb %%xmm3,%%xmm3,%%xmm3 \n" + + "vpunpcklqdq %%xmm4,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%[dst_ptr]) \n" + "vmovq %%xmm3,0x10(%[dst_ptr]) \n" + + "99: \n" + "vzeroupper \n" + : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), + [dst_width] "+rm"(dst_width) + : [src_stride] "r"(src_stride), [kShuf01] "m"(kShuf01), + [kShuf11] "m"(kShuf11), [kShuf21] "m"(kShuf21), [kMadd01] "m"(kMadd01), + [kMadd11] "m"(kMadd11), [kRound34] "m"(kRound34), [kMadd21] "m"(kMadd21) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); +} +#endif // HAS_SCALEROWDOWN34_AVX2 + +#ifdef HAS_SCALEROWDOWN38_AVX2 +void ScaleRowDown38_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + asm volatile( + "vmovdqu %3,%%xmm4 \n" + "vinserti128 $0x1,%4,%%ymm4,%%ymm4 \n" + "sub $0x18,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm4,%%ymm1,%%ymm1 \n" + "vextracti128 $0x1,%%ymm0,%%xmm2 \n" + "vextracti128 $0x1,%%ymm1,%%xmm3 \n" + "vpor %%xmm2,%%xmm0,%%xmm0 \n" + "vpor %%xmm3,%%xmm1,%%xmm1 \n" + "vmovq %%xmm0,(%1) \n" + "vpsrldq $0x8,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,0x8(%1) \n" + "vmovq %%xmm1,0xc(%1) \n" + "vpsrldq $0x8,%%xmm1,%%xmm1 \n" + "vmovd %%xmm1,0x14(%1) \n" + "lea 0x18(%1),%1 \n" + "sub $0x18,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x18,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vextracti128 $0x1,%%ymm0,%%xmm2 \n" + "vpor %%xmm2,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0,(%1) \n" + "vpsrldq $0x8,%%xmm0,%%xmm0 \n" + "vmovd %%xmm0,0x8(%1) \n" + + "99: \n" + "vzeroupper \n" : "+r"(src_ptr), // %0 "+r"(dst_ptr), // %1 "+r"(dst_width) // %2 : "m"(kShuf38a), // %3 "m"(kShuf38b) // %4 - : "memory", "cc", "xmm0", "xmm1", "xmm4", "xmm5"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); } -void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { asm volatile( - "movdqa %[kShufAb0],%%xmm2 \n" - "movdqa %[kShufAb1],%%xmm3 \n" - "movdqa %[kShufAb2],%%xmm4 \n" - "movdqa %[kScaleAb2],%%xmm5 \n" + "vbroadcasti128 %[kShufAb0],%%ymm2 \n" + "vbroadcasti128 %[kShufAb1],%%ymm3 \n" + "vbroadcasti128 %[kShufAb2],%%ymm4 \n" + "vbroadcasti128 %[kScaleAb2],%%ymm5 \n" + "vpxor %%ymm7,%%ymm7,%%ymm7 \n" + + LABELALIGN "1: \n" - "movdqu (%[src_ptr]),%%xmm0 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm1 \n" - "lea 0x10(%[src_ptr]),%[src_ptr] \n" - "pavgb %%xmm1,%%xmm0 \n" - "movdqa %%xmm0,%%xmm1 \n" - "pshufb %%xmm2,%%xmm1 \n" - "movdqa %%xmm0,%%xmm6 \n" - "pshufb %%xmm3,%%xmm6 \n" - "paddusw %%xmm6,%%xmm1 \n" - "pshufb %%xmm4,%%xmm0 \n" - "paddusw %%xmm0,%%xmm1 \n" - "pmulhuw %%xmm5,%%xmm1 \n" - "packuswb %%xmm1,%%xmm1 \n" - "movd %%xmm1,(%[dst_ptr]) \n" - "psrlq $0x10,%%xmm1 \n" - "movd %%xmm1,0x2(%[dst_ptr]) \n" - "lea 0x6(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x6,%[dst_width] \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm1 \n" + "lea 0x20(%[src_ptr]),%[src_ptr] \n" + "vpavgb %%ymm1,%%ymm0,%%ymm0 \n" + "vpshufb %%ymm2,%%ymm0,%%ymm1 \n" + "vpshufb %%ymm3,%%ymm0,%%ymm6 \n" + "vpaddusw %%ymm6,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vpaddusw %%ymm0,%%ymm1,%%ymm1 \n" + "vpmulhuw %%ymm5,%%ymm1,%%ymm1 \n" + "vpackuswb %%ymm7,%%ymm1,%%ymm1 \n" + "vextracti128 $0x1,%%ymm1,%%xmm0 \n" + "vpslldq $0x6,%%xmm0,%%xmm0 \n" + "vpor %%xmm0,%%xmm1,%%xmm1 \n" + "vmovq %%xmm1,(%[dst_ptr]) \n" + "vpsrldq $0x8,%%xmm1,%%xmm1 \n" + "vmovd %%xmm1,0x8(%[dst_ptr]) \n" + "lea 0xc(%[dst_ptr]),%[dst_ptr] \n" + "sub $0xc,%[dst_width] \n" "jg 1b \n" + "vzeroupper \n" : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) + [dst_width] "+rm"(dst_width) : [src_stride] "r"(src_stride), [kShufAb0] "m"(kShufAb0), [kShufAb1] "m"(kShufAb1), [kShufAb2] "m"(kShufAb2), [kScaleAb2] "m"(kScaleAb2) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", + "xmm7"); } -void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { asm volatile( - "movdqa %[kShufAc],%%xmm2 \n" - "movdqa %[kShufAc3],%%xmm3 \n" - "movdqa %[kScaleAc33],%%xmm4 \n" - "pxor %%xmm5,%%xmm5 \n" + "vbroadcasti128 %[kShufAc],%%ymm2 \n" + "vbroadcasti128 %[kShufAc3],%%ymm3 \n" + "vbroadcasti128 %[kScaleAc33],%%ymm4 \n" + "vpxor %%ymm5,%%ymm5,%%ymm5 \n" + + LABELALIGN "1: \n" - "movdqu (%[src_ptr]),%%xmm0 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],1),%%xmm6 \n" - "movhlps %%xmm0,%%xmm1 \n" - "movhlps %%xmm6,%%xmm7 \n" - "punpcklbw %%xmm5,%%xmm0 \n" - "punpcklbw %%xmm5,%%xmm1 \n" - "punpcklbw %%xmm5,%%xmm6 \n" - "punpcklbw %%xmm5,%%xmm7 \n" - "paddusw %%xmm6,%%xmm0 \n" - "paddusw %%xmm7,%%xmm1 \n" - "movdqu 0x00(%[src_ptr],%[src_stride],2),%%xmm6 \n" - "lea 0x10(%[src_ptr]),%[src_ptr] \n" - "movhlps %%xmm6,%%xmm7 \n" - "punpcklbw %%xmm5,%%xmm6 \n" - "punpcklbw %%xmm5,%%xmm7 \n" - "paddusw %%xmm6,%%xmm0 \n" - "paddusw %%xmm7,%%xmm1 \n" - "movdqa %%xmm0,%%xmm6 \n" - "psrldq $0x2,%%xmm0 \n" - "paddusw %%xmm0,%%xmm6 \n" - "psrldq $0x2,%%xmm0 \n" - "paddusw %%xmm0,%%xmm6 \n" - "pshufb %%xmm2,%%xmm6 \n" - "movdqa %%xmm1,%%xmm7 \n" - "psrldq $0x2,%%xmm1 \n" - "paddusw %%xmm1,%%xmm7 \n" - "psrldq $0x2,%%xmm1 \n" - "paddusw %%xmm1,%%xmm7 \n" - "pshufb %%xmm3,%%xmm7 \n" - "paddusw %%xmm7,%%xmm6 \n" - "pmulhuw %%xmm4,%%xmm6 \n" - "packuswb %%xmm6,%%xmm6 \n" - "movd %%xmm6,(%[dst_ptr]) \n" - "psrlq $0x10,%%xmm6 \n" - "movd %%xmm6,0x2(%[dst_ptr]) \n" - "lea 0x6(%[dst_ptr]),%[dst_ptr] \n" - "sub $0x6,%[dst_width] \n" + "vmovdqu (%[src_ptr]),%%ymm0 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],1),%%ymm6 \n" + "vpunpckhbw %%ymm5,%%ymm0,%%ymm1 \n" + "vpunpcklbw %%ymm5,%%ymm0,%%ymm0 \n" + "vpunpckhbw %%ymm5,%%ymm6,%%ymm7 \n" + "vpunpcklbw %%ymm5,%%ymm6,%%ymm6 \n" + "vpaddusw %%ymm6,%%ymm0,%%ymm0 \n" + "vpaddusw %%ymm7,%%ymm1,%%ymm1 \n" + "vmovdqu 0x00(%[src_ptr],%[src_stride],2),%%ymm6 \n" + "lea 0x20(%[src_ptr]),%[src_ptr] \n" + "vpunpckhbw %%ymm5,%%ymm6,%%ymm7 \n" + "vpunpcklbw %%ymm5,%%ymm6,%%ymm6 \n" + "vpaddusw %%ymm6,%%ymm0,%%ymm0 \n" + "vpaddusw %%ymm7,%%ymm1,%%ymm1 \n" + "vpsrldq $0x2,%%ymm0,%%ymm6 \n" + "vpaddusw %%ymm0,%%ymm6,%%ymm6 \n" + "vpsrldq $0x4,%%ymm0,%%ymm0 \n" + "vpaddusw %%ymm0,%%ymm6,%%ymm6 \n" + "vpshufb %%ymm2,%%ymm6,%%ymm6 \n" + "vpsrldq $0x2,%%ymm1,%%ymm7 \n" + "vpaddusw %%ymm1,%%ymm7,%%ymm7 \n" + "vpsrldq $0x4,%%ymm1,%%ymm1 \n" + "vpaddusw %%ymm1,%%ymm7,%%ymm7 \n" + "vpshufb %%ymm3,%%ymm7,%%ymm7 \n" + "vpaddusw %%ymm7,%%ymm6,%%ymm6 \n" + "vpmulhuw %%ymm4,%%ymm6,%%ymm6 \n" + "vpackuswb %%ymm5,%%ymm6,%%ymm6 \n" + "vextracti128 $0x1,%%ymm6,%%xmm0 \n" + "vpslldq $0x6,%%xmm0,%%xmm0 \n" + "vpor %%xmm0,%%xmm6,%%xmm6 \n" + "vmovq %%xmm6,(%[dst_ptr]) \n" + "vpsrldq $0x8,%%xmm6,%%xmm6 \n" + "vmovd %%xmm6,0x8(%[dst_ptr]) \n" + "lea 0xc(%[dst_ptr]),%[dst_ptr] \n" + "sub $0xc,%[dst_width] \n" "jg 1b \n" + "vzeroupper \n" : [src_ptr] "+r"(src_ptr), [dst_ptr] "+r"(dst_ptr), - [dst_width] "+r"(dst_width) + [dst_width] "+rm"(dst_width) : [src_stride] "r"(src_stride), [kShufAc] "m"(kShufAc), [kShufAc3] "m"(kShufAc3), [kScaleAc33] "m"(kScaleAc33) : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7"); } +#endif // HAS_SCALEROWDOWN38_AVX2 static const uvec8 kLinearShuffleFar = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}; @@ -1245,8 +1513,13 @@ void ScaleRowUp2_Linear_16_SSE2(const uint16_t* src_ptr, "psrld $2,%%xmm0 \n" // 3/4*near+1/4*far (lo) "psrld $2,%%xmm1 \n" // 3/4*near+1/4*far (hi) - "packssdw %%xmm1,%%xmm0 \n" - "pshufd $0b11011000,%%xmm0,%%xmm0 \n" + "pshuflw $0x88,%%xmm0,%%xmm0 \n" + "pshufhw $0x88,%%xmm0,%%xmm0 \n" + "pshufd $0x88,%%xmm0,%%xmm0 \n" + "pshuflw $0x88,%%xmm1,%%xmm1 \n" + "pshufhw $0x88,%%xmm1,%%xmm1 \n" + "pshufd $0x88,%%xmm1,%%xmm1 \n" + "punpckldq %%xmm1,%%xmm0 \n" "movdqu %%xmm0,(%1) \n" "lea 0x8(%0),%0 \n" @@ -1335,11 +1608,21 @@ void ScaleRowUp2_Bilinear_16_SSE2(const uint16_t* src_ptr, "paddd %%xmm1,%%xmm2 \n" // 9 3 3 1 + 8 (2, hi) "psrld $4,%%xmm2 \n" // ^ div by 16 (2, hi) - "packssdw %%xmm0,%%xmm4 \n" - "pshufd $0b11011000,%%xmm4,%%xmm4 \n" + "pshuflw $0x88,%%xmm4,%%xmm4 \n" + "pshufhw $0x88,%%xmm4,%%xmm4 \n" + "pshufd $0x88,%%xmm4,%%xmm4 \n" + "pshuflw $0x88,%%xmm0,%%xmm0 \n" + "pshufhw $0x88,%%xmm0,%%xmm0 \n" + "pshufd $0x88,%%xmm0,%%xmm0 \n" + "punpckldq %%xmm0,%%xmm4 \n" "movdqu %%xmm4,(%1) \n" // store above - "packssdw %%xmm2,%%xmm5 \n" - "pshufd $0b11011000,%%xmm5,%%xmm5 \n" + "pshuflw $0x88,%%xmm5,%%xmm5 \n" + "pshufhw $0x88,%%xmm5,%%xmm5 \n" + "pshufd $0x88,%%xmm5,%%xmm5 \n" + "pshuflw $0x88,%%xmm2,%%xmm2 \n" + "pshufhw $0x88,%%xmm2,%%xmm2 \n" + "pshufd $0x88,%%xmm2,%%xmm2 \n" + "punpckldq %%xmm2,%%xmm5 \n" "movdqu %%xmm5,(%1,%4,2) \n" // store below "lea 0x8(%0),%0 \n" @@ -1970,10 +2253,12 @@ void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, } #endif // HAS_SCALEADDROW_AVX512BW -#ifdef HAS_SCALEADDCOLS_AVX512BW +#if defined(HAS_SCALEADDCOLS1_AVX512BW) || defined(HAS_SCALEADDCOLS2_AVX512BW) static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; +#endif +#ifdef HAS_SCALEADDCOLS1_AVX512BW void ScaleAddCols1_AVX512BW(int dst_width, int boxheight, int x, @@ -2223,7 +2508,9 @@ void ScaleAddCols1_AVX512BW(int dst_width, : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "k1", "k2", "k3", "k4"); } +#endif // HAS_SCALEADDCOLS1_AVX512BW +#ifdef HAS_SCALEADDCOLS2_AVX512BW void ScaleAddCols2_AVX512BW(int dst_width, int boxheight, int x, @@ -2581,110 +2868,114 @@ void ScaleAddCols2_AVX512BW(int dst_width, "xmm17", "xmm18", "xmm19", "xmm20", "xmm21", "xmm22", "xmm23", "k1", "k2", "k3", "k4"); } -#endif // HAS_SCALEADDCOLS_AVX512BW +#endif // HAS_SCALEADDCOLS2_AVX512BW -static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040, - 0x0040, 0x0040, 0x0040, 0x0040}; +#ifdef HAS_SCALEFILTERCOLS_AVX2 +static const int32_t kStep0_7_Filter[8] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uvec8 kShufFracCols = {0, 0, 4, 4, 8, 8, 12, 12, + 128, 128, 128, 128, 128, 128, 128, 128}; -// Bilinear column filtering. SSSE3 version. -void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, - const uint8_t* src_ptr, - int dst_width, - int x, - int dx) { - intptr_t x0, x1, temp_pixel; - asm volatile( - "movd %6,%%xmm2 \n" - "movd %7,%%xmm3 \n" - "movl $0x04040000,%k2 \n" - "movd %k2,%%xmm5 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "psrlw $0x9,%%xmm6 \n" // 0x007f007f - "pcmpeqb %%xmm7,%%xmm7 \n" // 0x00ff mask for - // extracting 'a' - "psrlw $8,%%xmm7 \n" +// Bilinear column filtering. AVX2 version. +void ScaleFilterCols_AVX2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx) { + int n = dst_width & ~7; + int r = dst_width & 7; + if (n > 0) { + SIMD_ALIGNED(int32_t xi_buf[8]); + int32_t* xi_ptr = xi_buf; + uintptr_t tmp; + int loop_n = n; + asm volatile( + "vpbroadcastd %[x],%%ymm2 \n" + "vpbroadcastd %[dx],%%ymm3 \n" + "vpmulld %[kStep],%%ymm3,%%ymm0 \n" + "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" + "vpslld $0x3,%%ymm3,%%ymm3 \n" + "vbroadcasti128 %[kShufFrac],%%ymm5 \n" + "vpcmpeqb %%xmm6,%%xmm6,%%xmm6 \n" + "vpsrlw $0x8,%%xmm6,%%xmm7 \n" // 0x00ff + "vpsrlw $0x9,%%xmm6,%%xmm6 \n" // 0x007f - "pextrw $0x1,%%xmm2,%k3 \n" - "subl $0x2,%5 \n" - "jl 29f \n" - "movdqa %%xmm2,%%xmm0 \n" - "paddd %%xmm3,%%xmm0 \n" - "punpckldq %%xmm0,%%xmm2 \n" - "punpckldq %%xmm3,%%xmm3 \n" - "paddd %%xmm3,%%xmm3 \n" - "pextrw $0x3,%%xmm2,%k4 \n" - - LABELALIGN - "2: \n" - "movdqa %%xmm2,%%xmm1 \n" - "paddd %%xmm3,%%xmm2 \n" - "movzwl 0x00(%1,%3,1),%k2 \n" - "movd %k2,%%xmm0 \n" - "psrlw $0x9,%%xmm1 \n" - "movzwl 0x00(%1,%4,1),%k2 \n" - "movd %k2,%%xmm4 \n" - "pshufb %%xmm5,%%xmm1 \n" - "punpcklwd %%xmm4,%%xmm0 \n" - "pxor %%xmm6,%%xmm1 \n" - "movdqa %%xmm0,%%xmm4 \n" // Copy pixels. - "pmaddubsw %%xmm1,%%xmm0 \n" // a * (127 - f) + b * f - "pand %%xmm7,%%xmm4 \n" // Extract left pixels - // 'a'. - "paddw %%xmm4,%%xmm0 \n" // a * 128 + f * (b - a) - "pextrw $0x1,%%xmm2,%k3 \n" - "paddw %8,%%xmm0 \n" // add rounding bias 64. - "pextrw $0x3,%%xmm2,%k4 \n" - "psrlw $0x7,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movd %%xmm0,%k2 \n" - "mov %w2,(%0) \n" - "lea 0x2(%0),%0 \n" - "subl $0x2,%5 \n" - "jge 2b \n" - - LABELALIGN - "29: \n" - "addl $0x1,%5 \n" - "jl 99f \n" - "movzwl 0x00(%1,%3,1),%k2 \n" - "movd %k2,%%xmm0 \n" - "psrlw $0x9,%%xmm2 \n" - "pshufb %%xmm5,%%xmm2 \n" - "pxor %%xmm6,%%xmm2 \n" - "movdqa %%xmm0,%%xmm4 \n" // Copy pixels. - "pmaddubsw %%xmm2,%%xmm0 \n" // a * (127 - f) + b * f - "pand %%xmm7,%%xmm4 \n" // Extract left pixel 'a'. - "paddw %%xmm4,%%xmm0 \n" // a * 128 + f * (b - a) - "paddw %8,%%xmm0 \n" // add rounding bias 64. - "psrlw $0x7,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movd %%xmm0,%k2 \n" - "mov %b2,(%0) \n" - "99: \n" - : "+r"(dst_ptr), // %0 - "+r"(src_ptr), // %1 - "=&a"(temp_pixel), // %2 - "=&r"(x0), // %3 - "=&r"(x1), // %4 + LABELALIGN + "1: \n" + "vpsrad $0x10,%%ymm2,%%ymm0 \n" + "vmovdqu %%ymm0,(%[xi]) \n" + "vpsrlw $0x9,%%ymm2,%%ymm1 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" + "vpermq $0xd8,%%ymm1,%%ymm1 \n" + "vpxor %%xmm6,%%xmm1,%%xmm1 \n" #if defined(__x86_64__) - "+rm"(dst_width) // %5 + "movslq 0x00(%[xi]),%q[tmp] \n" + "vpinsrw $0x0,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x04(%[xi]),%q[tmp] \n" + "vpinsrw $0x1,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x08(%[xi]),%q[tmp] \n" + "vpinsrw $0x2,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x0c(%[xi]),%q[tmp] \n" + "vpinsrw $0x3,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x10(%[xi]),%q[tmp] \n" + "vpinsrw $0x4,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x14(%[xi]),%q[tmp] \n" + "vpinsrw $0x5,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x18(%[xi]),%q[tmp] \n" + "vpinsrw $0x6,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" + "movslq 0x1c(%[xi]),%q[tmp] \n" + "vpinsrw $0x7,0x00(%[src],%q[tmp],1),%%xmm0,%%xmm0 \n" #else - "+m"(dst_width) // %5 + "mov 0x00(%[xi]),%[tmp] \n" + "vpinsrw $0x0,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x04(%[xi]),%[tmp] \n" + "vpinsrw $0x1,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x08(%[xi]),%[tmp] \n" + "vpinsrw $0x2,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x0c(%[xi]),%[tmp] \n" + "vpinsrw $0x3,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x10(%[xi]),%[tmp] \n" + "vpinsrw $0x4,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x14(%[xi]),%[tmp] \n" + "vpinsrw $0x5,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x18(%[xi]),%[tmp] \n" + "vpinsrw $0x6,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" + "mov 0x1c(%[xi]),%[tmp] \n" + "vpinsrw $0x7,0x00(%[src],%[tmp],1),%%xmm0,%%xmm0 \n" #endif - : "rm"(x), // %6 - "rm"(dx), // %7 -#if defined(__x86_64__) - "x"(kFadd40) // %8 -#else - "m"(kFadd40) // %8 -#endif - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", - "xmm7"); + "vpand %%xmm7,%%xmm0,%%xmm4 \n" + "vpmaddubsw %%xmm1,%%xmm0,%%xmm0 \n" + "vpaddw %%xmm4,%%xmm0,%%xmm0 \n" + "vpsubw %%xmm6,%%xmm7,%%xmm4 \n" + "vpsrlw $0x1,%%xmm4,%%xmm4 \n" // 0x0040 + "vpaddw %%xmm4,%%xmm0,%%xmm0 \n" + "vpsrlw $0x7,%%xmm0,%%xmm0 \n" + "vpackuswb %%xmm0,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0,(%[dst]) \n" + "lea 0x8(%[dst]),%[dst] \n" + "subl $0x8,%[n] \n" + "jg 1b \n" + "vzeroupper \n" + : [dst] "+r"(dst_ptr), [n] "+rm"(loop_n), [tmp] "=&r"(tmp) + : [src] "r"(src_ptr), [xi] "r"(xi_ptr), [x] "m"(x), [dx] "m"(dx), + [kStep] "m"(kStep0_7_Filter), [kShufFrac] "m"(kShufFracCols) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + int xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[j] = (uint8_t)(a + ((((x & 0xffff) >> 9) * (b - a) + 0x40) >> 7)); + x = (int)((uint32_t)x + (uint32_t)dx); + } } +#endif // HAS_SCALEFILTERCOLS_AVX2 -// Reads 4 pixels, duplicates them and writes 8 pixels. -// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned. -void ScaleColsUp2_SSE2(uint8_t* dst_ptr, +#ifdef HAS_SCALECOLSUP2_AVX2 +// Reads 16 pixels, duplicates them and writes 32 pixels (plus 16/8 tails). +void ScaleColsUp2_AVX2(uint8_t* dst_ptr, const uint8_t* src_ptr, int dst_width, int x, @@ -2692,40 +2983,67 @@ void ScaleColsUp2_SSE2(uint8_t* dst_ptr, (void)x; (void)dx; asm volatile( + "sub $0x20,%2 \n" + "jl 2f \n" + + LABELALIGN "1: \n" - "movdqu (%1),%%xmm0 \n" + "vmovdqu (%1),%%xmm0 \n" "lea 0x10(%1),%1 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpcklbw %%xmm0,%%xmm0 \n" - "punpckhbw %%xmm1,%%xmm1 \n" - "movdqu %%xmm0,(%0) \n" - "movdqu %%xmm1,0x10(%0) \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpcklbw %%ymm0,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%0) \n" "lea 0x20(%0),%0 \n" "sub $0x20,%2 \n" - "jg 1b \n" + "jge 1b \n" + "2: \n" + "add $0x20,%2 \n" + "jle 99f \n" + "test $0x10,%2 \n" + "je 3f \n" + "vmovq (%1),%%xmm0 \n" + "lea 0x8(%1),%1 \n" + "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n" + "vmovdqu %%xmm0,(%0) \n" + "lea 0x10(%0),%0 \n" + + "3: \n" + "test $0x8,%2 \n" + "je 99f \n" + "vmovd (%1),%%xmm0 \n" + "vpunpcklbw %%xmm0,%%xmm0,%%xmm0 \n" + "vmovq %%xmm0,(%0) \n" + + "99: \n" + "vzeroupper \n" : "+r"(dst_ptr), // %0 "+r"(src_ptr), // %1 "+r"(dst_width) // %2 : - : "memory", "cc", "xmm0", "xmm1"); + : "memory", "cc", "xmm0"); } +#endif // HAS_SCALECOLSUP2_AVX2 -void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, +#ifdef HAS_SCALEARGBROWDOWN2_AVX2 +void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { (void)src_stride; asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "lea 0x20(%0),%0 \n" - "shufps $0xdd,%%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x4,%2 \n" + "vmovups (%0),%%ymm0 \n" + "vmovups 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovups %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x8,%2 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_argb), // %0 "+r"(dst_argb), // %1 "+r"(dst_width) // %2 @@ -2733,24 +3051,26 @@ void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, : "memory", "cc", "xmm0", "xmm1"); } -void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { (void)src_stride; asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "lea 0x20(%0),%0 \n" - "movdqa %%xmm0,%%xmm2 \n" - "shufps $0x88,%%xmm1,%%xmm0 \n" - "shufps $0xdd,%%xmm1,%%xmm2 \n" - "pavgb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x4,%2 \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm0,%%ymm2,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x8,%2 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_argb), // %0 "+r"(dst_argb), // %1 "+r"(dst_width) // %2 @@ -2758,188 +3078,165 @@ void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, : "memory", "cc", "xmm0", "xmm1", "xmm2"); } -void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb, +void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { asm volatile( + LABELALIGN "1: \n" - "movdqu (%0),%%xmm0 \n" - "movdqu 0x10(%0),%%xmm1 \n" - "movdqu 0x00(%0,%3,1),%%xmm2 \n" - "movdqu 0x10(%0,%3,1),%%xmm3 \n" - "lea 0x20(%0),%0 \n" - "pavgb %%xmm2,%%xmm0 \n" - "pavgb %%xmm3,%%xmm1 \n" - "movdqa %%xmm0,%%xmm2 \n" - "shufps $0x88,%%xmm1,%%xmm0 \n" - "shufps $0xdd,%%xmm1,%%xmm2 \n" - "pavgb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%1) \n" - "lea 0x10(%1),%1 \n" - "sub $0x4,%2 \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "vmovdqu 0x00(%0,%3,1),%%ymm2 \n" + "vmovdqu 0x20(%0,%3,1),%%ymm3 \n" + "lea 0x40(%0),%0 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm0,%%ymm2,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x8,%2 \n" "jg 1b \n" + "vzeroupper \n" : "+r"(src_argb), // %0 "+r"(dst_argb), // %1 "+r"(dst_width) // %2 : "r"(src_stride) // %3 : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); } +#endif // HAS_SCALEARGBROWDOWN2_AVX2 -// Reads 4 pixels at a time. -// Alignment requirement: dst_argb 16 byte aligned. -void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb, +#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2 +static const int32_t kStep0_7_Even[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + +// Reads 8 pixels at a time. +void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width) { - ptrdiff_t src_stepx_x4 = (ptrdiff_t)(src_stepx); - ptrdiff_t src_stepx_x12; + ptrdiff_t src_stepx_x32 = (ptrdiff_t)src_stepx * 32; (void)src_stride; asm volatile( - "lea 0x00(,%1,4),%1 \n" - "lea 0x00(%1,%1,2),%4 \n" + "vpbroadcastd %[stepx],%%ymm2 \n" + "vpmulld %[kStep],%%ymm2,%%ymm2 \n" LABELALIGN "1: \n" - "movd (%0),%%xmm0 \n" - "movd 0x00(%0,%1,1),%%xmm1 \n" - "punpckldq %%xmm1,%%xmm0 \n" - "movd 0x00(%0,%1,2),%%xmm2 \n" - "movd 0x00(%0,%4,1),%%xmm3 \n" - "lea 0x00(%0,%1,4),%0 \n" - "punpckldq %%xmm3,%%xmm2 \n" - "punpcklqdq %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" + "vpcmpeqd %%ymm3,%%ymm3,%%ymm3 \n" + "vpgatherdd %%ymm3,(%[src],%%ymm2,4),%%ymm0 \n" + "add %[step32],%[src] \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "sub $0x8,%[width] \n" "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(src_stepx_x4), // %1 - "+r"(dst_argb), // %2 - "+r"(dst_width), // %3 - "=&r"(src_stepx_x12) // %4 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + "vzeroupper \n" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [width] "+rm"(dst_width) + : [step32] "r"(src_stepx_x32), [stepx] "m"(src_stepx), + [kStep] "m"(kStep0_7_Even) + : "memory", "cc", "xmm0", "xmm2", "xmm3"); } -// Blends four 2x2 to 4x1. -// Alignment requirement: dst_argb 16 byte aligned. -void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb, +// Blends eight 2x2 to 8x1. +void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb, ptrdiff_t src_stride, int src_stepx, uint8_t* dst_argb, int dst_width) { - ptrdiff_t src_stepx_x4 = (ptrdiff_t)(src_stepx); - ptrdiff_t src_stepx_x12; + const uint8_t* row1 = src_argb + src_stride; + ptrdiff_t step4_bytes = (ptrdiff_t)src_stepx * 16; asm volatile( - "lea 0x00(,%1,4),%1 \n" - "lea 0x00(%1,%1,2),%4 \n" - "lea 0x00(%0,%5,1),%5 \n" + "vmovd %[stepx],%%xmm4 \n" + "vpshufd $0x0,%%xmm4,%%xmm4 \n" + "vpmulld %[kStep],%%xmm4,%%xmm4 \n" LABELALIGN "1: \n" - "movq (%0),%%xmm0 \n" - "movhps 0x00(%0,%1,1),%%xmm0 \n" - "movq 0x00(%0,%1,2),%%xmm1 \n" - "movhps 0x00(%0,%4,1),%%xmm1 \n" - "lea 0x00(%0,%1,4),%0 \n" - "movq (%5),%%xmm2 \n" - "movhps 0x00(%5,%1,1),%%xmm2 \n" - "movq 0x00(%5,%1,2),%%xmm3 \n" - "movhps 0x00(%5,%4,1),%%xmm3 \n" - "lea 0x00(%5,%1,4),%5 \n" - "pavgb %%xmm2,%%xmm0 \n" - "pavgb %%xmm3,%%xmm1 \n" - "movdqa %%xmm0,%%xmm2 \n" - "shufps $0x88,%%xmm1,%%xmm0 \n" - "shufps $0xdd,%%xmm1,%%xmm2 \n" - "pavgb %%xmm2,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%3 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpgatherdq %%ymm5,(%[row0],%%xmm4,4),%%ymm0 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpgatherdq %%ymm5,(%[row1],%%xmm4,4),%%ymm2 \n" + "add %[step4],%[row0] \n" + "add %[step4],%[row1] \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpgatherdq %%ymm5,(%[row0],%%xmm4,4),%%ymm1 \n" + "vpcmpeqd %%ymm5,%%ymm5,%%ymm5 \n" + "vpgatherdq %%ymm5,(%[row1],%%xmm4,4),%%ymm3 \n" + "add %[step4],%[row0] \n" + "add %[step4],%[row1] \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vshufps $0x88,%%ymm1,%%ymm0,%%ymm2 \n" + "vshufps $0xdd,%%ymm1,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm0,%%ymm2,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "subl $0x8,%[width] \n" "jg 1b \n" - : "+r"(src_argb), // %0 - "+r"(src_stepx_x4), // %1 - "+r"(dst_argb), // %2 - "+rm"(dst_width), // %3 - "=&r"(src_stepx_x12), // %4 - "+r"(src_stride) // %5 - : - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3"); + "vzeroupper \n" + : [row0] "+r"(src_argb), [row1] "+r"(row1), [dst] "+r"(dst_argb), + [width] "+rm"(dst_width) + : [step4] "r"(step4_bytes), [stepx] "m"(src_stepx), + [kStep] "m"(kStep0_7_Even) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); } +#endif // HAS_SCALEARGBROWDOWNEVEN_AVX2 -void ScaleARGBCols_SSE2(uint8_t* dst_argb, +#ifdef HAS_SCALEARGBCOLS_AVX2 +static const int32_t kStep0_7_Cols[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + +void ScaleARGBCols_AVX2(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, int dx) { - intptr_t x0, x1; - asm volatile( - "movd %5,%%xmm2 \n" - "movd %6,%%xmm3 \n" - "pshufd $0x0,%%xmm2,%%xmm2 \n" - "pshufd $0x11,%%xmm3,%%xmm0 \n" - "paddd %%xmm0,%%xmm2 \n" - "paddd %%xmm3,%%xmm3 \n" - "pshufd $0x5,%%xmm3,%%xmm0 \n" - "paddd %%xmm0,%%xmm2 \n" - "paddd %%xmm3,%%xmm3 \n" - "pshufd $0x0,%%xmm3,%%xmm3 \n" - "pextrw $0x1,%%xmm2,%k0 \n" - "pextrw $0x3,%%xmm2,%k1 \n" - "cmp $0x0,%4 \n" - "jl 99f \n" - "sub $0x4,%4 \n" - "jl 49f \n" + int n = dst_width & ~7; + int r = dst_width & 7; + if (n > 0) { + int loop_n = n; + asm volatile( + "vpbroadcastd %[x],%%ymm2 \n" + "vpbroadcastd %[dx],%%ymm3 \n" + "vpmulld %[kStep],%%ymm3,%%ymm0 \n" + "vpaddd %%ymm0,%%ymm2,%%ymm2 \n" + "vpslld $0x3,%%ymm3,%%ymm3 \n" - LABELALIGN - "40: \n" - "movd 0x00(%3,%0,4),%%xmm0 \n" - "movd 0x00(%3,%1,4),%%xmm1 \n" - "pextrw $0x5,%%xmm2,%k0 \n" - "pextrw $0x7,%%xmm2,%k1 \n" - "paddd %%xmm3,%%xmm2 \n" - "punpckldq %%xmm1,%%xmm0 \n" - "movd 0x00(%3,%0,4),%%xmm1 \n" - "movd 0x00(%3,%1,4),%%xmm4 \n" - "pextrw $0x1,%%xmm2,%k0 \n" - "pextrw $0x3,%%xmm2,%k1 \n" - "punpckldq %%xmm4,%%xmm1 \n" - "punpcklqdq %%xmm1,%%xmm0 \n" - "movdqu %%xmm0,(%2) \n" - "lea 0x10(%2),%2 \n" - "sub $0x4,%4 \n" - "jge 40b \n" - - "49: \n" - "test $0x2,%4 \n" - "je 29f \n" - "movd 0x00(%3,%0,4),%%xmm0 \n" - "movd 0x00(%3,%1,4),%%xmm1 \n" - "pextrw $0x5,%%xmm2,%k0 \n" - "punpckldq %%xmm1,%%xmm0 \n" - "movq %%xmm0,(%2) \n" - "lea 0x8(%2),%2 \n" - "29: \n" - "test $0x1,%4 \n" - "je 99f \n" - "movd 0x00(%3,%0,4),%%xmm0 \n" - "movd %%xmm0,(%2) \n" - "99: \n" - : "=&a"(x0), // %0 - "=&d"(x1), // %1 - "+r"(dst_argb), // %2 - "+r"(src_argb), // %3 - "+r"(dst_width) // %4 - : "rm"(x), // %5 - "rm"(dx) // %6 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); + LABELALIGN + "1: \n" + "vpsrad $0x10,%%ymm2,%%ymm0 \n" + "vpaddd %%ymm3,%%ymm2,%%ymm2 \n" + "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n" + "vpgatherdd %%ymm4,(%[src],%%ymm0,4),%%ymm1 \n" + "vmovdqu %%ymm1,(%[dst]) \n" + "lea 0x20(%[dst]),%[dst] \n" + "subl $0x8,%[n] \n" + "jg 1b \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [n] "+rm"(loop_n) + : [src] "r"(src_argb), [x] "m"(x), [dx] "m"(dx), + [kStep] "m"(kStep0_7_Cols) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + const uint8_t* src = src_argb + (ptrdiff_t)(x >> 16) * 4; + dst_argb[0] = src[0]; + dst_argb[1] = src[1]; + dst_argb[2] = src[2]; + dst_argb[3] = src[3]; + dst_argb += 4; + x = (int)((uint32_t)x + (uint32_t)dx); + } } +#endif // HAS_SCALEARGBCOLS_AVX2 +#ifdef HAS_SCALEARGBCOLSUP2_AVX2 // Reads 4 pixels, duplicates them and writes 8 pixels. -// Alignment requirement: src_argb 16 byte aligned, dst_argb 16 byte aligned. -void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, +void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb, const uint8_t* src_argb, int dst_width, int x, @@ -2947,24 +3244,27 @@ void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, (void)x; (void)dx; asm volatile( + LABELALIGN "1: \n" - "movdqu (%1),%%xmm0 \n" + "vmovdqu (%1),%%xmm0 \n" "lea 0x10(%1),%1 \n" - "movdqa %%xmm0,%%xmm1 \n" - "punpckldq %%xmm0,%%xmm0 \n" - "punpckhdq %%xmm1,%%xmm1 \n" - "movdqu %%xmm0,(%0) \n" - "movdqu %%xmm1,0x10(%0) \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vpunpckldq %%ymm0,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%0) \n" "lea 0x20(%0),%0 \n" "sub $0x8,%2 \n" "jg 1b \n" - + "vzeroupper \n" : "+r"(dst_argb), // %0 "+r"(src_argb), // %1 "+r"(dst_width) // %2 : - : "memory", "cc", "xmm0", "xmm1"); + : "memory", "cc", "xmm0"); } +#endif // HAS_SCALEARGBCOLSUP2_AVX2 + +#ifdef HAS_SCALEARGBFILTERCOLS_AVX2 +static const int32_t kStep0_3_ARGB[4] = {0, 1, 2, 3}; // Shuffle table for arranging 2 pixels into pairs for pmaddubsw static const uvec8 kShuffleColARGB = { @@ -2972,77 +3272,77 @@ static const uvec8 kShuffleColARGB = { 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel }; -// Shuffle table for duplicating 2 fractions into 8 bytes each -static const uvec8 kShuffleFractions = { - 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, +// Shuffle table for duplicating 4 fractions across 2x128-bit lanes +static const uint8_t kShuffleFractionsAVX2[32] = { + 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, + 8u, 8u, 8u, 8u, 8u, 8u, 8u, 8u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, }; -// Bilinear row filtering combines 4x2 -> 4x1. SSSE3 version -void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx) { - intptr_t x0, x1; - asm volatile( - "movdqa %[kShuffleColARGB],%%xmm4 \n" - "movdqa %[kShuffleFractions],%%xmm5 \n" - "movd %[x],%%xmm2 \n" - "movd %[dx],%%xmm3 \n" - "pcmpeqb %%xmm6,%%xmm6 \n" - "psrlw $0x9,%%xmm6 \n" - "pextrw $0x1,%%xmm2,%k[x0] \n" - "sub $0x2,%[dst_width] \n" - "jl 29f \n" - "movdqa %%xmm2,%%xmm0 \n" - "paddd %%xmm3,%%xmm0 \n" - "punpckldq %%xmm0,%%xmm2 \n" - "punpckldq %%xmm3,%%xmm3 \n" - "paddd %%xmm3,%%xmm3 \n" - "pextrw $0x3,%%xmm2,%k[x1] \n" +// Bilinear row filtering combines 4x2 -> 4x1. AVX2 version. +void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + int n = dst_width & ~3; + int r = dst_width & 3; + if (n > 0) { + int loop_n = n; + asm volatile( + "vbroadcasti128 %[kShufARGB],%%ymm4 \n" + "vmovdqu %[kShufFrac],%%ymm5 \n" + "vmovd %[x],%%xmm2 \n" + "vpshufd $0x0,%%xmm2,%%xmm2 \n" + "vmovd %[dx],%%xmm3 \n" + "vpshufd $0x0,%%xmm3,%%xmm3 \n" + "vpmulld %[kStep],%%xmm3,%%xmm0 \n" + "vpaddd %%xmm0,%%xmm2,%%xmm2 \n" + "vpslld $0x2,%%xmm3,%%xmm3 \n" + "vpcmpeqb %%ymm6,%%ymm6,%%ymm6 \n" + "vpsrlw $0x9,%%ymm6,%%ymm6 \n" - LABELALIGN - "2: \n" - "movdqa %%xmm2,%%xmm1 \n" - "paddd %%xmm3,%%xmm2 \n" - "movq 0x00(%[src_argb],%[x0],4),%%xmm0 \n" - "psrlw $0x9,%%xmm1 \n" - "movhps 0x00(%[src_argb],%[x1],4),%%xmm0 \n" - "pshufb %%xmm5,%%xmm1 \n" - "pshufb %%xmm4,%%xmm0 \n" - "pxor %%xmm6,%%xmm1 \n" - "pmaddubsw %%xmm1,%%xmm0 \n" - "psrlw $0x7,%%xmm0 \n" - "pextrw $0x1,%%xmm2,%k[x0] \n" - "pextrw $0x3,%%xmm2,%k[x1] \n" - "packuswb %%xmm0,%%xmm0 \n" - "movq %%xmm0,(%[dst_argb]) \n" - "lea 0x8(%[dst_argb]),%[dst_argb] \n" - "sub $0x2,%[dst_width] \n" - "jge 2b \n" - - LABELALIGN - "29: \n" - "add $0x1,%[dst_width] \n" - "jl 99f \n" - "psrlw $0x9,%%xmm2 \n" - "movq 0x00(%[src_argb],%[x0],4),%%xmm0 \n" - "pshufb %%xmm5,%%xmm2 \n" - "pshufb %%xmm4,%%xmm0 \n" - "pxor %%xmm6,%%xmm2 \n" - "pmaddubsw %%xmm2,%%xmm0 \n" - "psrlw $0x7,%%xmm0 \n" - "packuswb %%xmm0,%%xmm0 \n" - "movd %%xmm0,(%[dst_argb]) \n" - - LABELALIGN "99: \n" - - : [dst_argb] "+r"(dst_argb), [src_argb] "+r"(src_argb), - [dst_width] "+rm"(dst_width), [x0] "=&r"(x0), [x1] "=&r"(x1) - : [x] "rm"(x), [dx] "rm"(dx), [kShuffleColARGB] "m"(kShuffleColARGB), - [kShuffleFractions] "m"(kShuffleFractions) - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6"); + LABELALIGN + "1: \n" + "vpsrad $0x10,%%xmm2,%%xmm1 \n" + "vpcmpeqd %%ymm7,%%ymm7,%%ymm7 \n" + "vpgatherdq %%ymm7,(%[src],%%xmm1,4),%%ymm0 \n" + "vpsrlw $0x9,%%xmm2,%%xmm1 \n" + "vpaddd %%xmm3,%%xmm2,%%xmm2 \n" + "vinserti128 $0x1,%%xmm1,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm5,%%ymm1,%%ymm1 \n" + "vpshufb %%ymm4,%%ymm0,%%ymm0 \n" + "vpxor %%ymm6,%%ymm1,%%ymm1 \n" + "vpmaddubsw %%ymm1,%%ymm0,%%ymm0 \n" + "vpsrlw $0x7,%%ymm0,%%ymm0 \n" + "vpackuswb %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "subl $0x4,%[n] \n" + "jg 1b \n" + "vzeroupper \n" + : [dst] "+r"(dst_argb), [n] "+rm"(loop_n) + : [src] "r"(src_argb), [x] "m"(x), [dx] "m"(dx), + [kStep] "m"(kStep0_3_ARGB), [kShufARGB] "m"(kShuffleColARGB), + [kShufFrac] "m"(kShuffleFractionsAVX2) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", + "xmm6", "xmm7"); + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + ptrdiff_t xi = (ptrdiff_t)(x >> 16) * 4; + int xf = (x >> 9) & 0x7f; + const uint8_t* a = src_argb + xi; + const uint8_t* b = a + 4; + dst_argb[0] = (uint8_t)((a[0] * (0x7f ^ xf) + b[0] * xf) >> 7); + dst_argb[1] = (uint8_t)((a[1] * (0x7f ^ xf) + b[1] * xf) >> 7); + dst_argb[2] = (uint8_t)((a[2] * (0x7f ^ xf) + b[2] * xf) >> 7); + dst_argb[3] = (uint8_t)((a[3] * (0x7f ^ xf) + b[3] * xf) >> 7); + dst_argb += 4; + x = (int)((uint32_t)x + (uint32_t)dx); + } } +#endif // HAS_SCALEARGBFILTERCOLS_AVX2 // Divide num by div and return as 16.16 fixed point result. int FixedDiv_X86(int num, int div) { @@ -3076,59 +3376,108 @@ int FixedDiv1_X86(int num, int div) { return num; } -#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3) || \ - defined(HAS_SCALEUVROWDOWN2BOX_AVX2) +#ifdef HAS_SCALEUVROWDOWN2_AVX2 +void ScaleUVRowDown2_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + asm volatile( + "sub $0x10,%2 \n" + "jl 2f \n" + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm0 \n" + "vpsrld $0x10,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vpsrld $0x10,%%ymm0,%%ymm0 \n" + "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%1) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "xmm0", "xmm1"); +} +#endif // HAS_SCALEUVROWDOWN2_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2 +void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + asm volatile( + "vpcmpeqd %%ymm4,%%ymm4,%%ymm4 \n" + "vpsrld $0x10,%%ymm4,%%ymm4 \n" + "sub $0x10,%2 \n" + "jl 2f \n" + + LABELALIGN + "1: \n" + "vmovdqu (%0),%%ymm0 \n" + "vmovdqu 0x20(%0),%%ymm1 \n" + "lea 0x40(%0),%0 \n" + "vpsrld $0x10,%%ymm0,%%ymm2 \n" + "vpsrld $0x10,%%ymm1,%%ymm3 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vpavgb %%ymm3,%%ymm1,%%ymm1 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpand %%ymm4,%%ymm1,%%ymm1 \n" + "vpackusdw %%ymm1,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%ymm0,(%1) \n" + "lea 0x20(%1),%1 \n" + "sub $0x10,%2 \n" + "jge 1b \n" + + "2: \n" + "add $0x10,%2 \n" + "jle 99f \n" + "vmovdqu (%0),%%ymm0 \n" + "vpsrld $0x10,%%ymm0,%%ymm2 \n" + "vpavgb %%ymm2,%%ymm0,%%ymm0 \n" + "vpand %%ymm4,%%ymm0,%%ymm0 \n" + "vpackusdw %%ymm0,%%ymm0,%%ymm0 \n" + "vpermq $0xd8,%%ymm0,%%ymm0 \n" + "vmovdqu %%xmm0,(%1) \n" + + "99: \n" + "vzeroupper \n" + : "+r"(src_uv), // %0 + "+r"(dst_uv), // %1 + "+r"(dst_width) // %2 + : + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4"); +} +#endif // HAS_SCALEUVROWDOWN2LINEAR_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2 // Shuffle table for splitting UV into upper and lower part of register. static const uvec8 kShuffleSplitUV = {0u, 2u, 4u, 6u, 8u, 10u, 12u, 14u, 1u, 3u, 5u, 7u, 9u, 11u, 13u, 15u}; static const uvec8 kShuffleMergeUV = {0u, 8u, 2u, 10u, 4u, 12u, 6u, 14u, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}; -#endif -#ifdef HAS_SCALEUVROWDOWN2BOX_SSSE3 - -void ScaleUVRowDown2Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - asm volatile( - "pcmpeqb %%xmm4,%%xmm4 \n" // 01010101 - "psrlw $0xf,%%xmm4 \n" - "packuswb %%xmm4,%%xmm4 \n" - "pxor %%xmm5, %%xmm5 \n" // zero - "movdqa %4,%%xmm1 \n" // split shuffler - "movdqa %5,%%xmm3 \n" // merge shuffler - - LABELALIGN - "1: \n" - "movdqu (%0),%%xmm0 \n" // 8 UV row 0 - "movdqu 0x00(%0,%3,1),%%xmm2 \n" // 8 UV row 1 - "lea 0x10(%0),%0 \n" - "pshufb %%xmm1,%%xmm0 \n" // uuuuvvvv - "pshufb %%xmm1,%%xmm2 \n" - "pmaddubsw %%xmm4,%%xmm0 \n" // horizontal add - "pmaddubsw %%xmm4,%%xmm2 \n" - "paddw %%xmm2,%%xmm0 \n" // vertical add - "psrlw $0x1,%%xmm0 \n" // round - "pavgw %%xmm5,%%xmm0 \n" - "pshufb %%xmm3,%%xmm0 \n" // merge uv - "movq %%xmm0,(%1) \n" - "lea 0x8(%1),%1 \n" // 4 UV - "sub $0x4,%2 \n" - "jg 1b \n" - : "+r"(src_ptr), // %0 - "+r"(dst_ptr), // %1 - "+r"(dst_width) // %2 - : "r"(src_stride), // %3 - "m"(kShuffleSplitUV), // %4 - "m"(kShuffleMergeUV) // %5 - : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5"); -} -#endif // HAS_SCALEUVROWDOWN2BOX_SSSE3 - -#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2 void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, @@ -3169,6 +3518,40 @@ void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr, } #endif // HAS_SCALEUVROWDOWN2BOX_AVX2 +#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2 +void ScaleUVRowDownEven_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_uv, + int dst_width) { + ptrdiff_t step = (ptrdiff_t)src_stepx * 2; + ptrdiff_t step3 = step * 3; + (void)src_stride; + asm volatile( + "vpxor %%xmm0,%%xmm0,%%xmm0 \n" + + LABELALIGN + "1: \n" + "vpinsrw $0x0,(%[src]),%%xmm0,%%xmm0 \n" + "vpinsrw $0x1,0x00(%[src],%[step],1),%%xmm0,%%xmm0 \n" + "vpinsrw $0x2,0x00(%[src],%[step],2),%%xmm0,%%xmm0 \n" + "vpinsrw $0x3,0x00(%[src],%[step3],1),%%xmm0,%%xmm0 \n" + "lea 0x00(%[src],%[step],4),%[src] \n" + "vpinsrw $0x4,(%[src]),%%xmm0,%%xmm0 \n" + "vpinsrw $0x5,0x00(%[src],%[step],1),%%xmm0,%%xmm0 \n" + "vpinsrw $0x6,0x00(%[src],%[step],2),%%xmm0,%%xmm0 \n" + "vpinsrw $0x7,0x00(%[src],%[step3],1),%%xmm0,%%xmm0 \n" + "lea 0x00(%[src],%[step],4),%[src] \n" + "vmovdqu %%xmm0,(%[dst]) \n" + "lea 0x10(%[dst]),%[dst] \n" + "subl $0x8,%[width] \n" + "jg 1b \n" + : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [width] "+rm"(dst_width) + : [step] "r"(step), [step3] "r"(step3) + : "memory", "cc", "xmm0"); +} +#endif // HAS_SCALEUVROWDOWNEVEN_AVX2 + static const uvec8 kUVLinearMadd31 = {3, 1, 3, 1, 1, 3, 1, 3, 3, 1, 3, 1, 1, 3, 1, 3}; diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 324822a0e..706f27a80 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -84,25 +84,25 @@ static void ScaleUVDown2(int src_width, assert(dx == 65536 * 2); // Test scale factor of 2. assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2. // Advance to odd row, even column. - if (filtering == kFilterBilinear) { + if (filtering) { src_uv += (y >> 16) * src_stride + (x >> 16) * 2; } else { src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2; } -#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && filtering) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_SSSE3; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_SSSE3; - } - } -#endif -#if defined(HAS_SCALEUVROWDOWN2BOX_AVX2) - if (TestCpuFlag(kCpuHasAVX2) && filtering) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2; +#if defined(HAS_SCALEUVROWDOWN2_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleUVRowDown2 = + filtering == kFilterNone + ? ScaleUVRowDown2_Any_AVX2 + : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_AVX2 + : ScaleUVRowDown2Box_Any_AVX2); if (IS_ALIGNED(dst_width, 8)) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_AVX2; + ScaleUVRowDown2 = + filtering == kFilterNone + ? ScaleUVRowDown2_AVX2 + : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_AVX2 + : ScaleUVRowDown2Box_AVX2); } } #endif @@ -148,23 +148,48 @@ static void ScaleUVDown2(int src_width, } #endif -// This code is not enabled. Only box filter is available at this time. -#if defined(HAS_SCALEUVROWDOWN2_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ScaleUVRowDown2 = - filtering == kFilterNone - ? ScaleUVRowDown2_Any_SSSE3 - : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_Any_SSSE3 - : ScaleUVRowDown2Box_Any_SSSE3); - if (IS_ALIGNED(dst_width, 2)) { - ScaleUVRowDown2 = - filtering == kFilterNone - ? ScaleUVRowDown2_SSSE3 - : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_SSSE3 - : ScaleUVRowDown2Box_SSSE3); - } + if (filtering == kFilterLinear) { + src_stride = 0; + } + for (j = 0; j < dst_height; ++j) { + ScaleUVRowDown2(src_uv, src_stride, dst_uv, dst_width); + src_uv += row_stride; + dst_uv += dst_stride; + } +} + +static void ScaleUVDown2_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy, + enum FilterMode filtering) { + int j; + ptrdiff_t row_stride = src_stride * (dy >> 16); + void (*ScaleUVRowDown2)(const uint16_t* src_uv, ptrdiff_t src_stride, + uint16_t* dst_uv, int dst_width) = + filtering == kFilterNone + ? ScaleUVRowDown2_16_C + : (filtering == kFilterLinear ? ScaleUVRowDown2Linear_16_C + : ScaleUVRowDown2Box_16_C); + (void)src_width; + (void)src_height; + (void)dx; + assert(dx == 65536 * 2); // Test scale factor of 2. + assert((dy & 0x1ffff) == 0); // Test vertical scale is multiple of 2. + // Advance to odd row, even column. + if (filtering) { + src_uv += (y >> 16) * src_stride + (x >> 16) * 2; + } else { + src_uv += (y >> 16) * src_stride + ((x >> 16) - 1) * 2; } -#endif if (filtering == kFilterLinear) { src_stride = 0; @@ -211,14 +236,6 @@ static int ScaleUVDown4Box(int src_width, assert(dx == 65536 * 4); // Test scale factor of 4. assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4. -#if defined(HAS_SCALEUVROWDOWN2BOX_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_SSSE3; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVRowDown2 = ScaleUVRowDown2Box_SSSE3; - } - } -#endif #if defined(HAS_SCALEUVROWDOWN2BOX_AVX2) if (TestCpuFlag(kCpuHasAVX2)) { ScaleUVRowDown2 = ScaleUVRowDown2Box_Any_AVX2; @@ -262,6 +279,49 @@ static int ScaleUVDown4Box(int src_width, free_aligned_buffer_64(row); return 0; } + +static int ScaleUVDown4Box_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy) { + int j; + // Allocate 2 rows of 16-bit UV. + const int row_stride_words = (dst_width * 2 * 2 + 15) & ~15; + align_buffer_64(row, row_stride_words * 2 * (int)sizeof(uint16_t)); + if (!row) + return 1; + uint16_t* row16 = (uint16_t*)row; + ptrdiff_t row_stride = src_stride * (dy >> 16); + void (*ScaleUVRowDown2)(const uint16_t* src_uv, ptrdiff_t src_stride, + uint16_t* dst_uv, int dst_width) = + ScaleUVRowDown2Box_16_C; + // Advance to odd row, even column. + src_uv += (y >> 16) * src_stride + (x >> 16) * 2; + (void)src_width; + (void)src_height; + (void)dx; + assert(dx == 65536 * 4); // Test scale factor of 4. + assert((dy & 0x3ffff) == 0); // Test vertical scale is multiple of 4. + + for (j = 0; j < dst_height; ++j) { + ScaleUVRowDown2(src_uv, src_stride, row16, dst_width * 2); + ScaleUVRowDown2(src_uv + src_stride * 2, src_stride, + row16 + row_stride_words, dst_width * 2); + ScaleUVRowDown2(row16, row_stride_words, dst_uv, dst_width); + src_uv += row_stride; + dst_uv += dst_stride; + } + free_aligned_buffer_64(row); + return 0; +} #endif // HAS_SCALEUVDOWN4BOX // ScaleUV, any dimensions, box filter. @@ -493,6 +553,119 @@ static int ScaleUVBox(int src_width, } return 0; } + +static void ScaleUVAddCols1_16_C(int dst_width, + int boxheight, + int x, + int dx, + const uint32_t* src_uv, + uint16_t* dst_uv) { + int i, j; + int boxwidth = MIN1(dx >> 16); + uint32_t scaleval = 65536 / (boxwidth * boxheight); + x >>= 16; + for (i = 0; i < dst_width; ++i) { + uint32_t u = 0; + uint32_t v = 0; + for (j = 0; j < boxwidth; ++j) { + u += src_uv[(x + j) * 2]; + v += src_uv[(x + j) * 2 + 1]; + } + x += boxwidth; + *dst_uv++ = (uint16_t)((u * scaleval) >> 16); + *dst_uv++ = (uint16_t)((v * scaleval) >> 16); + } +} + +static void ScaleUVAddCols2_16_C(int dst_width, + int boxheight, + int x, + int dx, + const uint32_t* src_uv, + uint16_t* dst_uv) { + int i, j; + int minboxwidth = dx >> 16; + uint32_t scaletbl[2]; + scaletbl[0] = 65536 / (MIN1(minboxwidth) * boxheight); + scaletbl[1] = 65536 / (MIN1(minboxwidth + 1) * boxheight); + for (i = 0; i < dst_width; ++i) { + int ix = x >> 16; + uint32_t u = 0; + uint32_t v = 0; + int boxwidth; + uint32_t scaleval; + x += dx; + boxwidth = MIN1((x >> 16) - ix); + assert((boxwidth - minboxwidth == 0) || (boxwidth - minboxwidth == 1)); + scaleval = scaletbl[boxwidth - minboxwidth]; + for (j = 0; j < boxwidth; ++j) { + u += src_uv[(ix + j) * 2]; + v += src_uv[(ix + j) * 2 + 1]; + } + *dst_uv++ = (uint16_t)((u * scaleval) >> 16); + *dst_uv++ = (uint16_t)((v * scaleval) >> 16); + } +} + +static int ScaleUVBox_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy) { + int j, k; + const uint32_t max_y = (uint32_t)src_height << 16; + const int row_samples = src_width * 2; + { + // Row accumulator, two uint32_t per source UV pixel. + align_buffer_64(row32_buf, row_samples * (int)sizeof(uint32_t)); + if (!row32_buf) + return 1; + { + uint32_t* row32 = (uint32_t*)row32_buf; + void (*ScaleUVAddCols)(int dst_width, int boxheight, int x, int dx, + const uint32_t* src_uv, uint16_t* dst_uv) = + (dx & 0xffff) ? ScaleUVAddCols2_16_C : ScaleUVAddCols1_16_C; + void (*ScaleUVAddCols3)(int dst_width, int scaleval, + const uint32_t* src_uv, uint16_t* dst_uv) = + (dx == 0x30000) ? ScaleUVAddCols3_16_C : NULL; + void (*ScaleAddRow)(const uint16_t* src_ptr, uint32_t* dst_ptr, + int src_width) = ScaleAddRow_16_C; + + uint32_t uy = y; + for (j = 0; j < dst_height; ++j) { + int boxheight; + int iy = uy >> 16; + const uint16_t* src = src_uv + iy * src_stride; + uy += dy; + if (uy > max_y) { + uy = max_y; + } + boxheight = MIN1((int)(uy >> 16) - iy); + memset(row32, 0, row_samples * sizeof(uint32_t)); + for (k = 0; k < boxheight; ++k) { + ScaleAddRow(src, row32, row_samples); + src += src_stride; + } + if (ScaleUVAddCols3 && boxheight <= 21845) { + ScaleUVAddCols3(dst_width, 65536 / (3 * boxheight), + row32 + (ptrdiff_t)(x >> 16) * 2, dst_uv); + } else { + ScaleUVAddCols(dst_width, boxheight, x, dx, row32, dst_uv); + } + dst_uv += dst_stride; + } + } + free_aligned_buffer_64(row32_buf); + } + return 0; +} #endif // HAS_SCALEUVBOX // ScaleUV Even @@ -523,13 +696,11 @@ static void ScaleUVDownEven(int src_width, assert(IS_ALIGNED(src_width, 2)); assert(IS_ALIGNED(src_height, 2)); src_uv += (y >> 16) * src_stride + (x >> 16) * 2; -#if defined(HAS_SCALEUVROWDOWNEVEN_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3)) { - ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_SSSE3 - : ScaleUVRowDownEven_Any_SSSE3; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVRowDownEven = - filtering ? ScaleUVRowDownEvenBox_SSE2 : ScaleUVRowDownEven_SSSE3; +#if defined(HAS_SCALEUVROWDOWNEVEN_AVX2) + if (TestCpuFlag(kCpuHasAVX2) && !filtering) { + ScaleUVRowDownEven = ScaleUVRowDownEven_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + ScaleUVRowDownEven = ScaleUVRowDownEven_AVX2; } } #endif @@ -541,16 +712,6 @@ static void ScaleUVDownEven(int src_width, } } #endif // TODO(fbarchard): Enable Box filter -#if defined(HAS_SCALEUVROWDOWNEVENBOX_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ScaleUVRowDownEven = filtering ? ScaleUVRowDownEvenBox_Any_NEON - : ScaleUVRowDownEven_Any_NEON; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVRowDownEven = - filtering ? ScaleUVRowDownEvenBox_NEON : ScaleUVRowDownEven_NEON; - } - } -#endif #if defined(HAS_SCALEUVROWDOWNEVEN_RVV) || defined(HAS_SCALEUVROWDOWN4_RVV) if (TestCpuFlag(kCpuHasRVV) && !filtering) { #if defined(HAS_SCALEUVROWDOWNEVEN_RVV) @@ -573,6 +734,41 @@ static void ScaleUVDownEven(int src_width, dst_uv += dst_stride; } } + +static void ScaleUVDownEven_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy, + enum FilterMode filtering) { + int j; + int col_step = dx >> 16; + ptrdiff_t row_stride = (dy >> 16) * src_stride; + void (*ScaleUVRowDownEven)(const uint16_t* src_uv, ptrdiff_t src_stride, + int src_step, uint16_t* dst_uv, int dst_width) = + filtering ? ScaleUVRowDownEvenBox_16_C : ScaleUVRowDownEven_16_C; + (void)src_width; + (void)src_height; + assert(IS_ALIGNED(src_width, 2)); + assert(IS_ALIGNED(src_height, 2)); + src_uv += (y >> 16) * src_stride + (x >> 16) * 2; + + if (filtering == kFilterLinear) { + src_stride = 0; + } + for (j = 0; j < dst_height; ++j) { + ScaleUVRowDownEven(src_uv, src_stride, col_step, dst_uv, dst_width); + src_uv += row_stride; + dst_uv += dst_stride; + } +} #endif // Scale UV down with bilinear interpolation. @@ -674,19 +870,6 @@ static int ScaleUVBilinearDown(int src_width, InterpolateRow = InterpolateRow_WASMSIMD; } } -#endif -#if defined(HAS_SCALEUVFILTERCOLS_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleUVFilterCols = ScaleUVFilterCols_SSSE3; - } -#endif -#if defined(HAS_SCALEUVFILTERCOLS_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON; - if (IS_ALIGNED(dst_width, 4)) { - ScaleUVFilterCols = ScaleUVFilterCols_NEON; - } - } #endif // TODO(fbarchard): Consider not allocating row buffer for kFilterLinear. // Allocate a row of UV. @@ -720,6 +903,110 @@ static int ScaleUVBilinearDown(int src_width, } return 0; } + +static int ScaleUVBilinearDown_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy, + enum FilterMode filtering) { + int j; + void (*InterpolateRow)(uint16_t* dst_uv, const uint16_t* src_uv, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) = InterpolateRow_16_C; + void (*ScaleUVFilterCols)(uint16_t* dst_uv, const uint16_t* src_uv, + int dst_width, int x, int dx) = + (src_width >= 32768) ? ScaleUVFilterCols64_16_C : ScaleUVFilterCols_16_C; + int64_t xlast = x + (int64_t)(dst_width - 1) * dx; + int64_t xl = (dx >= 0) ? x : xlast; + int64_t xr = (dx >= 0) ? xlast : x; + int clip_src_width; + xl = (xl >> 16) & ~3; // Left edge aligned. + xr = (xr >> 16) + 1; // Right most pixel used. Bilinear uses 2 pixels. + xr = (xr + 1 + 3) & ~3; // 1 beyond 4 pixel aligned right most pixel. + if (xr > src_width) { + xr = src_width; + } + clip_src_width = (int)(xr - xl) * 2; // Width in uint16_t words. + src_uv += xl * 2; + x -= (int)(xl << 16); +#if defined(HAS_INTERPOLATEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_16_Any_AVX2; + if (IS_ALIGNED(clip_src_width, 16)) { + InterpolateRow = InterpolateRow_16_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_16_AVX512BW; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_16_Any_NEON; + if (IS_ALIGNED(clip_src_width, 8)) { + InterpolateRow = InterpolateRow_16_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_SME) + if (TestCpuFlag(kCpuHasSME)) { + InterpolateRow = InterpolateRow_16_SME; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + InterpolateRow = InterpolateRow_16_HVX; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_16_Any_WASMSIMD; + if (IS_ALIGNED(clip_src_width, 8)) { + InterpolateRow = InterpolateRow_16_WASMSIMD; + } + } +#endif + // Allocate a row of 16-bit UV. + { + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; + uint32_t uy = y; + align_buffer_64(row, clip_src_width * (int)sizeof(uint16_t)); + if (!row) + return 1; + uint16_t* row16 = (uint16_t*)row; + if (uy > max_y) { + uy = max_y; + } + for (j = 0; j < dst_height; ++j) { + int yi = uy >> 16; + const uint16_t* src = src_uv + yi * src_stride; + if (filtering == kFilterLinear) { + ScaleUVFilterCols(dst_uv, src, dst_width, x, dx); + } else { + int yf = (uy >> 8) & 255; + InterpolateRow(row16, src, src_stride, clip_src_width, yf); + ScaleUVFilterCols(dst_uv, row16, dst_width, x, dx); + } + dst_uv += dst_stride; + uy += dy; + if (uy > max_y) { + uy = max_y; + } + } + free_aligned_buffer_64(row); + } + return 0; +} #endif // Scale UV up with bilinear interpolation. @@ -820,39 +1107,8 @@ static int ScaleUVBilinearUp(int src_width, if (src_width >= 32768) { ScaleUVFilterCols = filtering ? ScaleUVFilterCols64_C : ScaleUVCols64_C; } -#if defined(HAS_SCALEUVFILTERCOLS_SSSE3) - if (filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleUVFilterCols = ScaleUVFilterCols_SSSE3; - } -#endif -#if defined(HAS_SCALEUVFILTERCOLS_NEON) - if (filtering && TestCpuFlag(kCpuHasNEON)) { - ScaleUVFilterCols = ScaleUVFilterCols_Any_NEON; - if (IS_ALIGNED(dst_width, 8)) { - ScaleUVFilterCols = ScaleUVFilterCols_NEON; - } - } -#endif -#if defined(HAS_SCALEUVCOLS_SSSE3) - if (!filtering && TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleUVFilterCols = ScaleUVCols_SSSE3; - } -#endif -#if defined(HAS_SCALEUVCOLS_NEON) - if (!filtering && TestCpuFlag(kCpuHasNEON)) { - ScaleUVFilterCols = ScaleUVCols_Any_NEON; - if (IS_ALIGNED(dst_width, 16)) { - ScaleUVFilterCols = ScaleUVCols_NEON; - } - } -#endif if (!filtering && src_width * 2 == dst_width && x < 0x8000) { ScaleUVFilterCols = ScaleUVColsUp2_C; -#if defined(HAS_SCALEUVCOLSUP2_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) { - ScaleUVFilterCols = ScaleUVColsUp2_SSSE3; - } -#endif } if (uy > max_y) { @@ -919,6 +1175,136 @@ static int ScaleUVBilinearUp(int src_width, } return 0; } + +static int ScaleUVBilinearUp_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy, + enum FilterMode filtering) { + assert(src_width > 0); + assert(src_height > 0); + assert(dst_width > 0); + assert(dst_height > 0); + assert(dy <= 65536); + + int j; + void (*InterpolateRow)(uint16_t* dst_uv, const uint16_t* src_uv, + ptrdiff_t src_stride, int dst_width, + int source_y_fraction) = InterpolateRow_16_C; + void (*ScaleUVFilterCols)(uint16_t* dst_uv, const uint16_t* src_uv, + int dst_width, int x, int dx) = + filtering ? ScaleUVFilterCols_16_C : ScaleUVCols_16_C; + const uint32_t max_y = (uint32_t)(src_height - 1) << 16; + uint32_t uy = y; +#if defined(HAS_INTERPOLATEROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + InterpolateRow = InterpolateRow_16_Any_AVX2; + if (IS_ALIGNED(dst_width, 8)) { + InterpolateRow = InterpolateRow_16_AVX2; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_AVX512BW) + if (TestCpuFlag(kCpuHasAVX512BW)) { + InterpolateRow = InterpolateRow_16_AVX512BW; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + InterpolateRow = InterpolateRow_16_Any_NEON; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_16_NEON; + } + } +#endif +#if defined(HAS_INTERPOLATEROW_16_SME) + if (TestCpuFlag(kCpuHasSME)) { + InterpolateRow = InterpolateRow_16_SME; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_HVX) + if (TestCpuFlag(kCpuHasHVX)) { + InterpolateRow = InterpolateRow_16_HVX; + } +#endif +#if defined(HAS_INTERPOLATEROW_16_WASMSIMD) + if (TestCpuFlag(kCpuHasWASMSIMD)) { + InterpolateRow = InterpolateRow_16_Any_WASMSIMD; + if (IS_ALIGNED(dst_width, 4)) { + InterpolateRow = InterpolateRow_16_WASMSIMD; + } + } +#endif + if (src_width >= 32768) { + ScaleUVFilterCols = + filtering ? ScaleUVFilterCols64_16_C : ScaleUVCols64_16_C; + } + if (!filtering && src_width * 2 == dst_width && x < 0x8000) { + ScaleUVFilterCols = ScaleUVColsUp2_16_C; + } + + if (uy > max_y) { + uy = max_y; + } + + { + int yi = uy >> 16; + const uint16_t* src = src_uv + yi * src_stride; + + // Allocate 2 rows of 16-bit UV. + const int row_stride_words = (dst_width * 2 + 15) & ~15; + align_buffer_64(row, row_stride_words * 2 * (int)sizeof(uint16_t)); + if (!row) + return 1; + + uint16_t* rowptr = (uint16_t*)row; + ptrdiff_t rowstride = row_stride_words; + int lasty = yi; + + ScaleUVFilterCols(rowptr, src, dst_width, x, dx); + if (src_height > 1) { + src += src_stride; + } + ScaleUVFilterCols(rowptr + rowstride, src, dst_width, x, dx); + if (src_height > 2) { + src += src_stride; + } + + for (j = 0; j < dst_height; ++j) { + if (uy > max_y) { + uy = max_y; + } + yi = uy >> 16; + if (yi != lasty) { + ScaleUVFilterCols(rowptr, src, dst_width, x, dx); + rowptr += rowstride; + rowstride = -rowstride; + lasty = yi; + if ((uy + 65536) < max_y) { + src += src_stride; + } + } + if (filtering == kFilterLinear) { + InterpolateRow(dst_uv, rowptr, 0, dst_width * 2, 0); + } else { + int yf = (uy >> 8) & 255; + InterpolateRow(dst_uv, rowptr, rowstride, dst_width * 2, yf); + } + dst_uv += dst_stride; + uy += dy; + } + free_aligned_buffer_64(row); + } + return 0; +} #endif // HAS_SCALEUVBILINEARUP // Scale UV, horizontally up by 2 times. @@ -1169,26 +1555,38 @@ static void ScaleUVSimple(int src_width, int x, int dx) = (src_width >= 32768) ? ScaleUVCols64_C : ScaleUVCols_C; (void)src_height; -#if defined(HAS_SCALEUVCOLS_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && src_width < 32768) { - ScaleUVCols = ScaleUVCols_SSSE3; - } -#endif -#if defined(HAS_SCALEUVCOLS_NEON) - if (TestCpuFlag(kCpuHasNEON)) { - ScaleUVCols = ScaleUVCols_Any_NEON; - if (IS_ALIGNED(dst_width, 8)) { - ScaleUVCols = ScaleUVCols_NEON; - } - } -#endif if (src_width * 2 == dst_width && x < 0x8000) { ScaleUVCols = ScaleUVColsUp2_C; -#if defined(HAS_SCALEUVCOLSUP2_SSSE3) - if (TestCpuFlag(kCpuHasSSSE3) && IS_ALIGNED(dst_width, 8)) { - ScaleUVCols = ScaleUVColsUp2_SSSE3; - } -#endif + } + + uint32_t uy = y; + for (j = 0; j < dst_height; ++j) { + int yi = uy >> 16; + ScaleUVCols(dst_uv, src_uv + yi * src_stride, dst_width, x, dx); + dst_uv += dst_stride; + uy += dy; + } +} + +static void ScaleUVSimple_16(int src_width, + int src_height, + int dst_width, + int dst_height, + ptrdiff_t src_stride, + ptrdiff_t dst_stride, + const uint16_t* src_uv, + uint16_t* dst_uv, + int x, + int dx, + int y, + int dy) { + int j; + void (*ScaleUVCols)(uint16_t* dst_uv, const uint16_t* src_uv, int dst_width, + int x, int dx) = + (src_width >= 32768) ? ScaleUVCols64_16_C : ScaleUVCols_16_C; + (void)src_height; + if (src_width * 2 == dst_width && x < 0x8000) { + ScaleUVCols = ScaleUVColsUp2_16_C; } uint32_t uy = y; @@ -1402,8 +1800,166 @@ int UVScale(const uint8_t* src_uv, dst_height, filtering); } +// Scale a 16-bit UV plane (from P010/P210) +static int ScaleUV_16(const uint16_t* src, + int src_stride, + int src_width, + int src_height, + uint16_t* dst, + int dst_stride, + int dst_width, + int dst_height, + int clip_x, + int clip_y, + int clip_width, + int clip_height, + enum FilterMode filtering) { + // Initial source x/y coordinate and step values as 16.16 fixed point. + int x = 0; + int y = 0; + int dx = 0; + int dy = 0; + const enum FilterMode orig_filtering = filtering; + // Simplify filtering when possible. + filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height, + filtering); + + // Negative src_height means invert the image. + if (src_height < 0) { + src_height = -src_height; + src = src + (src_height - 1) * (ptrdiff_t)src_stride; + src_stride = -src_stride; + } + ScaleSlope(src_width, src_height, dst_width, dst_height, filtering, &x, &y, + &dx, &dy); + if (clip_x) { + int64_t clipf = (int64_t)(clip_x)*dx; + x += (clipf & 0xffff); + src += (clipf >> 16) * 2; + dst += clip_x * 2; + } + if (clip_y) { + int64_t clipf = (int64_t)(clip_y)*dy; + y += (clipf & 0xffff); + src += (clipf >> 16) * (ptrdiff_t)src_stride; + dst += clip_y * (ptrdiff_t)dst_stride; + } + +#if HAS_SCALEUVBOX + // Exactly 4 goes to the faster ScaleUVDown4Box_16 below. Clipping is excluded + // because whole source rows are accumulated. + if (filtering == kFilterBox && !(dx == 0x40000 && dy == 0x40000) && + clip_x == 0 && clip_y == 0) { + return ScaleUVBox_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy); + } +#endif + + // Special case for integer step values. + if (((dx | dy) & 0xffff) == 0) { + if (!dx || !dy) { // 1 pixel wide and/or tall. + filtering = kFilterNone; + } else { + // Optimized even scale down. ie 2, 4, 6, 8, 10x. + if (!(dx & 0x10000) && !(dy & 0x10000)) { +#if HAS_SCALEUVDOWN2 + if (dx == 0x20000 && dy == 0x20000) { + // Optimized 1/2 downsample. + ScaleUVDown2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy, + filtering); + return 0; + } +#endif +#if HAS_SCALEUVDOWN4BOX + if (dx == 0x40000 && dy == 0x40000 && filtering == kFilterBox) { + // Optimized 1/4 box downsample. + return ScaleUVDown4Box_16(src_width, src_height, clip_width, + clip_height, src_stride, dst_stride, src, + dst, x, dx, y, dy); + } +#endif +#if HAS_SCALEUVDOWNEVEN + ScaleUVDownEven_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy, + filtering); + return 0; +#endif + } + // Optimized odd scale down. ie 3, 5, 7, 9x. + if ((dx & 0x10000) && (dy & 0x10000)) { + filtering = kFilterNone; +#ifdef HAS_UVCOPY + if (dx == 0x10000 && dy == 0x10000) { + // Straight copy. + return UVCopy_16( + src + (y >> 16) * (ptrdiff_t)src_stride + (x >> 16) * 2, + src_stride, dst, dst_stride, clip_width, clip_height); + } +#endif + } + } + } + + // ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone to avoid + // reading a second source column. The Up2 kernels clamp their last pixel and + // only read the 1 source column when dst_width <= 2, so restore Linear or + // Bilinear for 2x upscales (e.g., P010ToP410 and P210ToP410 on 1-2 pixel wide + // images) when the caller requested filtering. + if (orig_filtering != kFilterNone && filtering == kFilterNone && + src_width == 1 && dst_width <= 2 && clip_x == 0 && clip_y == 0) { + if (clip_height == src_height && (dst_width + 1) / 2 == src_width) { + ScaleUVLinearUp2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst); + return 0; + } + if ((clip_height + 1) / 2 == src_height && + (clip_width + 1) / 2 == src_width && orig_filtering != kFilterLinear) { + ScaleUVBilinearUp2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst); + return 0; + } + } + + // HAS_SCALEPLANEVERTICAL + if (dx == 0x10000 && (x & 0xffff) == 0) { + // Arbitrary scale vertically, but unscaled horizontally. + ScalePlaneVertical_16(src_height, clip_width, clip_height, src_stride, + dst_stride, src, dst, x, y, dy, /*wpp=*/2, filtering); + return 0; + } + if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) { + ScaleUVLinearUp2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst); + return 0; + } + if ((clip_height + 1) / 2 == src_height && + (clip_width + 1) / 2 == src_width && + (filtering == kFilterBilinear || filtering == kFilterBox)) { + ScaleUVBilinearUp2_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst); + return 0; + } +#if HAS_SCALEUVBILINEARUP + if (filtering && dy < 65536) { + return ScaleUVBilinearUp_16(src_width, src_height, clip_width, clip_height, + src_stride, dst_stride, src, dst, x, dx, y, dy, + filtering); + } +#endif +#if HAS_SCALEUVBILINEARDOWN + if (filtering) { + return ScaleUVBilinearDown_16(src_width, src_height, clip_width, + clip_height, src_stride, dst_stride, src, dst, + x, dx, y, dy, filtering); + } +#endif + ScaleUVSimple_16(src_width, src_height, clip_width, clip_height, src_stride, + dst_stride, src, dst, x, dx, y, dy); + return 0; +} + // Scale a 16 bit UV image. -// This function is currently incomplete, it can't handle all cases. LIBYUV_API int UVScale_16(const uint16_t* src_uv, int src_stride_uv, @@ -1414,65 +1970,14 @@ int UVScale_16(const uint16_t* src_uv, int dst_width, int dst_height, enum FilterMode filtering) { - int dy = 0; - if (!src_uv || src_width <= 0 || src_height == 0 || src_width > 32768 || src_height < -32768 || src_height > 32768 || !dst_uv || dst_width <= 0 || dst_height <= 0) { return -1; } - - // UV does not support box filter yet, but allow the user to pass it. - // Simplify filtering when possible. - filtering = ScaleFilterReduce(src_width, src_height, dst_width, dst_height, - filtering); - - // Negative src_height means invert the image. - if (src_height < 0) { - src_height = -src_height; - src_uv = src_uv + (src_height - 1) * (ptrdiff_t)src_stride_uv; - src_stride_uv = -src_stride_uv; - } - -#ifdef HAS_UVCOPY - if (!filtering && src_width == dst_width && (src_height % dst_height == 0)) { - if (dst_height == 1) { - return UVCopy_16( - src_uv + ((src_height - 1) / 2) * (ptrdiff_t)src_stride_uv, - src_stride_uv, dst_uv, dst_stride_uv, dst_width, dst_height); - } - dy = src_height / dst_height; - if (src_stride_uv > INT_MAX / dy) { - return -1; - } - return UVCopy_16(src_uv + ((dy - 1) / 2) * (ptrdiff_t)src_stride_uv, - dy * src_stride_uv, dst_uv, dst_stride_uv, dst_width, - dst_height); - } -#endif - - // ScaleFilterReduce reduces a 1 pixel wide source to kFilterNone, which has - // no 16 bit implementation other than the copy above. The Up2 kernels only - // read the pixels they need, so restore Linear or Bilinear for a 2x upscale - // so P010ToP410 and P210ToP410 work on 1 pixel wide images. - if (filtering == kFilterNone && src_width == 1 && dst_width <= 2) { - filtering = (src_height == dst_height) ? kFilterLinear : kFilterBilinear; - } - - if ((filtering == kFilterLinear) && ((dst_width + 1) / 2 == src_width)) { - ScaleUVLinearUp2_16(src_width, src_height, dst_width, dst_height, - src_stride_uv, dst_stride_uv, src_uv, dst_uv); - return 0; - } - - if ((dst_height + 1) / 2 == src_height && (dst_width + 1) / 2 == src_width && - (filtering == kFilterBilinear || filtering == kFilterBox)) { - ScaleUVBilinearUp2_16(src_width, src_height, dst_width, dst_height, - src_stride_uv, dst_stride_uv, src_uv, dst_uv); - return 0; - } - - return -1; + return ScaleUV_16(src_uv, src_stride_uv, src_width, src_height, dst_uv, + dst_stride_uv, dst_width, dst_height, 0, 0, dst_width, + dst_height, filtering); } #ifdef __cplusplus diff --git a/source/scale_win.cc b/source/scale_win.cc index f2ad7f078..d63877083 100644 --- a/source/scale_win.cc +++ b/source/scale_win.cc @@ -18,6 +18,7 @@ defined(LIBYUV_ENABLE_ROWWIN)) #include #include +#include #include #if defined(_MSC_VER) && !defined(__clang__) #include @@ -31,43 +32,9 @@ extern "C" { // This module is for 32 bit Visual C x86 #if !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) && \ - (!defined(__clang__) || defined(LIBYUV_ENABLE_ROWWIN)) - -// Offsets for source bytes 0 to 9 -static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9, - 128, 128, 128, 128, 128, 128, 128, 128}; - -// Offsets for source bytes 11 to 20 with 8 subtracted = 3 to 12. -static const uvec8 kShuf1 = {3, 4, 5, 7, 8, 9, 11, 12, - 128, 128, 128, 128, 128, 128, 128, 128}; - -// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. -static const uvec8 kShuf2 = {5, 7, 8, 9, 11, 12, 13, 15, - 128, 128, 128, 128, 128, 128, 128, 128}; - -// Offsets for source bytes 0 to 10 -static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10}; - -// Offsets for source bytes 10 to 21 with 8 subtracted = 3 to 13. -static const uvec8 kShuf11 = {2, 3, 4, 5, 5, 6, 6, 7, - 8, 9, 9, 10, 10, 11, 12, 13}; - -// Offsets for source bytes 21 to 31 with 16 subtracted = 5 to 31. -static const uvec8 kShuf21 = {5, 6, 6, 7, 8, 9, 9, 10, - 10, 11, 12, 13, 13, 14, 14, 15}; - -// Coefficients for source bytes 0 to 10 -static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2}; - -// Coefficients for source bytes 10 to 21 -static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1}; - -// Coefficients for source bytes 21 to 31 -static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; - -// Coefficients for source bytes 21 to 31 -static const vec16 kRound34 = {2, 2, 2, 2, 2, 2, 2, 2}; + !defined(__clang__) && !defined(LIBYUV_ENABLE_ROWWIN) +#ifdef HAS_SCALEROWDOWN2_SSSE3 // Reads 32 pixels, throws half away and writes 16 pixels. __declspec(naked) void ScaleRowDown2_SSSE3(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -172,122 +139,9 @@ __declspec(naked) void ScaleRowDown2Box_SSSE3(const uint8_t* src_ptr, ret } } +#endif // HAS_SCALEROWDOWN2_SSSE3 -#ifdef HAS_SCALEROWDOWN2_AVX2 -// Reads 64 pixels, throws half away and writes 32 pixels. -__declspec(naked) void ScaleRowDown2_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride ignored - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - - wloop: - vmovdqu ymm0, [eax] - vmovdqu ymm1, [eax + 32] - lea eax, [eax + 64] - vpsrlw ymm0, ymm0, 8 // isolate odd pixels. - vpsrlw ymm1, ymm1, 8 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], ymm0 - lea edx, [edx + 32] - sub ecx, 32 - jg wloop - - vzeroupper - ret - } -} - -// Blends 64x1 rectangle to 32x1. -__declspec(naked) void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - - vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b - vpsrlw ymm4, ymm4, 15 - vpackuswb ymm4, ymm4, ymm4 - vpxor ymm5, ymm5, ymm5 // constant 0 - - wloop: - vmovdqu ymm0, [eax] - vmovdqu ymm1, [eax + 32] - lea eax, [eax + 64] - vpmaddubsw ymm0, ymm0, ymm4 // horizontal add - vpmaddubsw ymm1, ymm1, ymm4 - vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2 - vpavgw ymm1, ymm1, ymm5 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], ymm0 - lea edx, [edx + 32] - sub ecx, 32 - jg wloop - - vzeroupper - ret - } -} - -// For rounding, average = (sum + 2) / 4 -// becomes average((sum >> 1), 0) -// Blends 64x2 rectangle to 32x1. -__declspec(naked) void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_ptr - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_ptr - mov ecx, [esp + 4 + 16] // dst_width - - vpcmpeqb ymm4, ymm4, ymm4 // '1' constant, 8b - vpsrlw ymm4, ymm4, 15 - vpackuswb ymm4, ymm4, ymm4 - vpxor ymm5, ymm5, ymm5 // constant 0 - - wloop: - vmovdqu ymm0, [eax] - vmovdqu ymm1, [eax + 32] - vmovdqu ymm2, [eax + esi] - vmovdqu ymm3, [eax + esi + 32] - lea eax, [eax + 64] - vpmaddubsw ymm0, ymm0, ymm4 // horizontal add - vpmaddubsw ymm1, ymm1, ymm4 - vpmaddubsw ymm2, ymm2, ymm4 - vpmaddubsw ymm3, ymm3, ymm4 - vpaddw ymm0, ymm0, ymm2 // vertical add - vpaddw ymm1, ymm1, ymm3 - vpsrlw ymm0, ymm0, 1 // (x + 2) / 4 = (x / 2 + 1) / 2 - vpsrlw ymm1, ymm1, 1 - vpavgw ymm0, ymm0, ymm5 // (x + 1) / 2 - vpavgw ymm1, ymm1, ymm5 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], ymm0 - lea edx, [edx + 32] - sub ecx, 32 - jg wloop - - pop esi - vzeroupper - ret - } -} -#endif // HAS_SCALEROWDOWN2_AVX2 - +#ifdef HAS_SCALEROWDOWN4_SSSE3 // Point samples 32 pixels to 8 pixels. __declspec(naked) void ScaleRowDown4_SSSE3(const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -377,755 +231,9 @@ __declspec(naked) void ScaleRowDown4Box_SSSE3(const uint8_t* src_ptr, ret } } +#endif // HAS_SCALEROWDOWN4_SSSE3 -#ifdef HAS_SCALEROWDOWN4_AVX2 -// Point samples 64 pixels to 16 pixels. -__declspec(naked) void ScaleRowDown4_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride ignored - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - vpcmpeqb ymm5, ymm5, ymm5 // generate mask 0x00ff0000 - vpsrld ymm5, ymm5, 24 - vpslld ymm5, ymm5, 16 - - wloop: - vmovdqu ymm0, [eax] - vmovdqu ymm1, [eax + 32] - lea eax, [eax + 64] - vpand ymm0, ymm0, ymm5 - vpand ymm1, ymm1, ymm5 - vpackuswb ymm0, ymm0, ymm1 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vpsrlw ymm0, ymm0, 8 - vpackuswb ymm0, ymm0, ymm0 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 16 - jg wloop - - vzeroupper - ret - } -} - -// Blends 64x4 rectangle to 16x1. -__declspec(naked) void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - push edi - mov eax, [esp + 8 + 4] // src_ptr - mov esi, [esp + 8 + 8] // src_stride - mov edx, [esp + 8 + 12] // dst_ptr - mov ecx, [esp + 8 + 16] // dst_width - lea edi, [esi + esi * 2] // src_stride * 3 - vpcmpeqb ymm4, ymm4, ymm4 // constant 0x0101 - vpsrlw ymm4, ymm4, 15 - vpsllw ymm5, ymm4, 3 // constant 0x0008 - vpackuswb ymm4, ymm4, ymm4 - - wloop: - vmovdqu ymm0, [eax] // average rows - vmovdqu ymm1, [eax + 32] - vmovdqu ymm2, [eax + esi] - vmovdqu ymm3, [eax + esi + 32] - vpmaddubsw ymm0, ymm0, ymm4 // horizontal add - vpmaddubsw ymm1, ymm1, ymm4 - vpmaddubsw ymm2, ymm2, ymm4 - vpmaddubsw ymm3, ymm3, ymm4 - vpaddw ymm0, ymm0, ymm2 // vertical add rows 0, 1 - vpaddw ymm1, ymm1, ymm3 - vmovdqu ymm2, [eax + esi * 2] - vmovdqu ymm3, [eax + esi * 2 + 32] - vpmaddubsw ymm2, ymm2, ymm4 - vpmaddubsw ymm3, ymm3, ymm4 - vpaddw ymm0, ymm0, ymm2 // add row 2 - vpaddw ymm1, ymm1, ymm3 - vmovdqu ymm2, [eax + edi] - vmovdqu ymm3, [eax + edi + 32] - lea eax, [eax + 64] - vpmaddubsw ymm2, ymm2, ymm4 - vpmaddubsw ymm3, ymm3, ymm4 - vpaddw ymm0, ymm0, ymm2 // add row 3 - vpaddw ymm1, ymm1, ymm3 - vphaddw ymm0, ymm0, ymm1 // mutates - vpermq ymm0, ymm0, 0xd8 // unmutate vphaddw - vpaddw ymm0, ymm0, ymm5 // + 8 for round - vpsrlw ymm0, ymm0, 4 // /32 for average of 4 * 4 - vpackuswb ymm0, ymm0, ymm0 - vpermq ymm0, ymm0, 0xd8 // unmutate vpackuswb - vmovdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 16 - jg wloop - - pop edi - pop esi - vzeroupper - ret - } -} -#endif // HAS_SCALEROWDOWN4_AVX2 - -// Point samples 32 pixels to 24 pixels. -// Produces three 8 byte values. For each 8 bytes, 16 bytes are read. -// Then shuffled to do the scaling. - -__declspec(naked) void ScaleRowDown34_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_ptr - // src_stride ignored - mov edx, [esp + 12] // dst_ptr - mov ecx, [esp + 16] // dst_width - movdqa xmm3, xmmword ptr kShuf0 - movdqa xmm4, xmmword ptr kShuf1 - movdqa xmm5, xmmword ptr kShuf2 - - wloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + 16] - lea eax, [eax + 32] - movdqa xmm2, xmm1 - palignr xmm1, xmm0, 8 - pshufb xmm0, xmm3 - pshufb xmm1, xmm4 - pshufb xmm2, xmm5 - movq qword ptr [edx], xmm0 - movq qword ptr [edx + 8], xmm1 - movq qword ptr [edx + 16], xmm2 - lea edx, [edx + 24] - sub ecx, 24 - jg wloop - - ret - } -} - -// Blends 32x2 rectangle to 24x1 -// Produces three 8 byte values. For each 8 bytes, 16 bytes are read. -// Then shuffled to do the scaling. - -// Register usage: -// xmm0 src_row 0 -// xmm1 src_row 1 -// xmm2 shuf 0 -// xmm3 shuf 1 -// xmm4 shuf 2 -// xmm5 madd 0 -// xmm6 madd 1 -// xmm7 kRound34 - -// Note that movdqa+palign may be better than movdqu. -__declspec(naked) void ScaleRowDown34_1_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_ptr - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_ptr - mov ecx, [esp + 4 + 16] // dst_width - movdqa xmm2, xmmword ptr kShuf01 - movdqa xmm3, xmmword ptr kShuf11 - movdqa xmm4, xmmword ptr kShuf21 - movdqa xmm5, xmmword ptr kMadd01 - movdqa xmm6, xmmword ptr kMadd11 - movdqa xmm7, xmmword ptr kRound34 - - wloop: - movdqu xmm0, [eax] // pixels 0..7 - movdqu xmm1, [eax + esi] - pavgb xmm0, xmm1 - pshufb xmm0, xmm2 - pmaddubsw xmm0, xmm5 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx], xmm0 - movdqu xmm0, [eax + 8] // pixels 8..15 - movdqu xmm1, [eax + esi + 8] - pavgb xmm0, xmm1 - pshufb xmm0, xmm3 - pmaddubsw xmm0, xmm6 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx + 8], xmm0 - movdqu xmm0, [eax + 16] // pixels 16..23 - movdqu xmm1, [eax + esi + 16] - lea eax, [eax + 32] - pavgb xmm0, xmm1 - pshufb xmm0, xmm4 - movdqa xmm1, xmmword ptr kMadd21 - pmaddubsw xmm0, xmm1 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx + 16], xmm0 - lea edx, [edx + 24] - sub ecx, 24 - jg wloop - - pop esi - ret - } -} - -// Note that movdqa+palign may be better than movdqu. -__declspec(naked) void ScaleRowDown34_0_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_ptr - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_ptr - mov ecx, [esp + 4 + 16] // dst_width - movdqa xmm2, xmmword ptr kShuf01 - movdqa xmm3, xmmword ptr kShuf11 - movdqa xmm4, xmmword ptr kShuf21 - movdqa xmm5, xmmword ptr kMadd01 - movdqa xmm6, xmmword ptr kMadd11 - movdqa xmm7, xmmword ptr kRound34 - - wloop: - movdqu xmm0, [eax] // pixels 0..7 - movdqu xmm1, [eax + esi] - pavgb xmm1, xmm0 - pavgb xmm0, xmm1 - pshufb xmm0, xmm2 - pmaddubsw xmm0, xmm5 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx], xmm0 - movdqu xmm0, [eax + 8] // pixels 8..15 - movdqu xmm1, [eax + esi + 8] - pavgb xmm1, xmm0 - pavgb xmm0, xmm1 - pshufb xmm0, xmm3 - pmaddubsw xmm0, xmm6 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx + 8], xmm0 - movdqu xmm0, [eax + 16] // pixels 16..23 - movdqu xmm1, [eax + esi + 16] - lea eax, [eax + 32] - pavgb xmm1, xmm0 - pavgb xmm0, xmm1 - pshufb xmm0, xmm4 - movdqa xmm1, xmmword ptr kMadd21 - pmaddubsw xmm0, xmm1 - paddsw xmm0, xmm7 - psrlw xmm0, 2 - packuswb xmm0, xmm0 - movq qword ptr [edx + 16], xmm0 - lea edx, [edx+24] - sub ecx, 24 - jg wloop - - pop esi - ret - } -} - -static const uvec16 kFadd40 = {0x0040, 0x0040, 0x0040, 0x0040, - 0x0040, 0x0040, 0x0040, 0x0040}; - -// Bilinear column filtering. SSSE3 version. -__declspec(naked) void ScaleFilterCols_SSSE3(uint8_t* dst_ptr, - const uint8_t* src_ptr, - int dst_width, - int x, - int dx) { - __asm { - push ebx - push esi - push edi - mov edi, [esp + 12 + 4] // dst_ptr - mov esi, [esp + 12 + 8] // src_ptr - mov ecx, [esp + 12 + 12] // dst_width - movd xmm2, [esp + 12 + 16] // x - movd xmm3, [esp + 12 + 20] // dx - mov eax, 0x04040000 // shuffle to line up fractions with pixel. - movd xmm5, eax - pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction. - psrlw xmm6, 9 - pcmpeqb xmm7, xmm7 // generate 0x00ff mask for extracting 'a' - psrlw xmm7, 8 - pextrw eax, xmm2, 1 // get x0 integer. preroll - sub ecx, 2 - jl xloop29 - - movdqa xmm0, xmm2 // x1 = x0 + dx - paddd xmm0, xmm3 - punpckldq xmm2, xmm0 // x0 x1 - punpckldq xmm3, xmm3 // dx dx - paddd xmm3, xmm3 // dx * 2, dx * 2 - pextrw edx, xmm2, 3 // get x1 integer. preroll - - // 2 Pixel loop. - xloop2: - movdqa xmm1, xmm2 // x0, x1 fractions. - paddd xmm2, xmm3 // x += dx - movzx ebx, word ptr [esi + eax] // 2 source x0 pixels - movd xmm0, ebx - psrlw xmm1, 9 // 7 bit fractions. - movzx ebx, word ptr [esi + edx] // 2 source x1 pixels - movd xmm4, ebx - pshufb xmm1, xmm5 // 0011 - punpcklwd xmm0, xmm4 - pxor xmm1, xmm6 // 0..7f and 7f..0 - movdqa xmm4, xmm0 // Copy pixels. - pmaddubsw xmm0, xmm1 // a * (127 - f) + b * f - pand xmm4, xmm7 // Extract left pixels 'a'. - paddw xmm0, xmm4 // a * 128 + f * (b - a) - pextrw eax, xmm2, 1 // get x0 integer. next iteration. - paddw xmm0, xmmword ptr kFadd40 // add rounding bias 64. - pextrw edx, xmm2, 3 // get x1 integer. next iteration. - psrlw xmm0, 7 // 8.7 fixed point to low 8 bits. - packuswb xmm0, xmm0 // 8 bits, 2 pixels. - movd ebx, xmm0 - mov [edi], bx - lea edi, [edi + 2] - sub ecx, 2 // 2 pixels - jge xloop2 - - xloop29: - add ecx, 2 - 1 - jl xloop99 - - // 1 pixel remainder - movzx ebx, word ptr [esi + eax] // 2 source x0 pixels - movd xmm0, ebx - psrlw xmm2, 9 // 7 bit fractions. - pshufb xmm2, xmm5 // 0011 - pxor xmm2, xmm6 // 0..7f and 7f..0 - movdqa xmm4, xmm0 // Copy pixels. - pmaddubsw xmm0, xmm2 // a * (127 - f) + b * f - pand xmm4, xmm7 // Extract left pixel 'a'. - paddw xmm0, xmm4 // a * 128 + f * (b - a) - paddw xmm0, xmmword ptr kFadd40 // add rounding bias 64. - psrlw xmm0, 7 // 8.7 fixed point to low 8 bits. - packuswb xmm0, xmm0 // 8 bits - movd ebx, xmm0 - mov [edi], bl - - xloop99: - - pop edi - pop esi - pop ebx - ret - } -} - -// Reads 16 pixels, duplicates them and writes 32 pixels. -__declspec(naked) void ScaleColsUp2_SSE2(uint8_t* dst_ptr, - const uint8_t* src_ptr, - int dst_width, - int x, - int dx) { - __asm { - mov edx, [esp + 4] // dst_ptr - mov eax, [esp + 8] // src_ptr - mov ecx, [esp + 12] // dst_width - - wloop: - movdqu xmm0, [eax] - lea eax, [eax + 16] - movdqa xmm1, xmm0 - punpcklbw xmm0, xmm0 - punpckhbw xmm1, xmm1 - movdqu [edx], xmm0 - movdqu [edx + 16], xmm1 - lea edx, [edx + 32] - sub ecx, 32 - jg wloop - - ret - } -} - -// Reads 8 pixels, throws half away and writes 4 even pixels (0, 2, 4, 6) -__declspec(naked) void ScaleARGBRowDown2_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_argb - // src_stride ignored - mov edx, [esp + 12] // dst_argb - mov ecx, [esp + 16] // dst_width - - wloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + 16] - lea eax, [eax + 32] - shufps xmm0, xmm1, 0xdd - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - ret - } -} - -// Blends 8x1 rectangle to 4x1. -__declspec(naked) void ScaleARGBRowDown2Linear_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - __asm { - mov eax, [esp + 4] // src_argb - // src_stride ignored - mov edx, [esp + 12] // dst_argb - mov ecx, [esp + 16] // dst_width - - wloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + 16] - lea eax, [eax + 32] - movdqa xmm2, xmm0 - shufps xmm0, xmm1, 0x88 // even pixels - shufps xmm2, xmm1, 0xdd // odd pixels - pavgb xmm0, xmm2 - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - ret - } -} - -// Blends 8x2 rectangle to 4x1. -__declspec(naked) void ScaleARGBRowDown2Box_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - uint8_t* dst_argb, - int dst_width) { - __asm { - push esi - mov eax, [esp + 4 + 4] // src_argb - mov esi, [esp + 4 + 8] // src_stride - mov edx, [esp + 4 + 12] // dst_argb - mov ecx, [esp + 4 + 16] // dst_width - - wloop: - movdqu xmm0, [eax] - movdqu xmm1, [eax + 16] - movdqu xmm2, [eax + esi] - movdqu xmm3, [eax + esi + 16] - lea eax, [eax + 32] - pavgb xmm0, xmm2 // average rows - pavgb xmm1, xmm3 - movdqa xmm2, xmm0 // average columns (8 to 4 pixels) - shufps xmm0, xmm1, 0x88 // even pixels - shufps xmm2, xmm1, 0xdd // odd pixels - pavgb xmm0, xmm2 - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - pop esi - ret - } -} - -// Reads 4 pixels at a time. -__declspec(naked) void ScaleARGBRowDownEven_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - int src_stepx, - uint8_t* dst_argb, - int dst_width) { - __asm { - push ebx - push edi - mov eax, [esp + 8 + 4] // src_argb - // src_stride ignored - mov ebx, [esp + 8 + 12] // src_stepx - mov edx, [esp + 8 + 16] // dst_argb - mov ecx, [esp + 8 + 20] // dst_width - lea ebx, [ebx * 4] - lea edi, [ebx + ebx * 2] - - wloop: - movd xmm0, [eax] - movd xmm1, [eax + ebx] - punpckldq xmm0, xmm1 - movd xmm2, [eax + ebx * 2] - movd xmm3, [eax + edi] - lea eax, [eax + ebx * 4] - punpckldq xmm2, xmm3 - punpcklqdq xmm0, xmm2 - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - pop edi - pop ebx - ret - } -} - -// Blends four 2x2 to 4x1. -__declspec(naked) void ScaleARGBRowDownEvenBox_SSE2(const uint8_t* src_argb, - ptrdiff_t src_stride, - int src_stepx, - uint8_t* dst_argb, - int dst_width) { - __asm { - push ebx - push esi - push edi - mov eax, [esp + 12 + 4] // src_argb - mov esi, [esp + 12 + 8] // src_stride - mov ebx, [esp + 12 + 12] // src_stepx - mov edx, [esp + 12 + 16] // dst_argb - mov ecx, [esp + 12 + 20] // dst_width - lea esi, [eax + esi] // row1 pointer - lea ebx, [ebx * 4] - lea edi, [ebx + ebx * 2] - - wloop: - movq xmm0, qword ptr [eax] // row0 4 pairs - movhps xmm0, qword ptr [eax + ebx] - movq xmm1, qword ptr [eax + ebx * 2] - movhps xmm1, qword ptr [eax + edi] - lea eax, [eax + ebx * 4] - movq xmm2, qword ptr [esi] // row1 4 pairs - movhps xmm2, qword ptr [esi + ebx] - movq xmm3, qword ptr [esi + ebx * 2] - movhps xmm3, qword ptr [esi + edi] - lea esi, [esi + ebx * 4] - pavgb xmm0, xmm2 // average rows - pavgb xmm1, xmm3 - movdqa xmm2, xmm0 // average columns (8 to 4 pixels) - shufps xmm0, xmm1, 0x88 // even pixels - shufps xmm2, xmm1, 0xdd // odd pixels - pavgb xmm0, xmm2 - movdqu [edx], xmm0 - lea edx, [edx + 16] - sub ecx, 4 - jg wloop - - pop edi - pop esi - pop ebx - ret - } -} - -// Column scaling unfiltered. SSE2 version. -__declspec(naked) void ScaleARGBCols_SSE2(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx) { - __asm { - push edi - push esi - mov edi, [esp + 8 + 4] // dst_argb - mov esi, [esp + 8 + 8] // src_argb - mov ecx, [esp + 8 + 12] // dst_width - movd xmm2, [esp + 8 + 16] // x - movd xmm3, [esp + 8 + 20] // dx - - pshufd xmm2, xmm2, 0 // x0 x0 x0 x0 - pshufd xmm0, xmm3, 0x11 // dx 0 dx 0 - paddd xmm2, xmm0 - paddd xmm3, xmm3 // 0, 0, 0, dx * 2 - pshufd xmm0, xmm3, 0x05 // dx * 2, dx * 2, 0, 0 - paddd xmm2, xmm0 // x3 x2 x1 x0 - paddd xmm3, xmm3 // 0, 0, 0, dx * 4 - pshufd xmm3, xmm3, 0 // dx * 4, dx * 4, dx * 4, dx * 4 - - pextrw eax, xmm2, 1 // get x0 integer. - pextrw edx, xmm2, 3 // get x1 integer. - - cmp ecx, 0 - jle xloop99 - sub ecx, 4 - jl xloop49 - - // 4 Pixel loop. - xloop4: - movd xmm0, [esi + eax * 4] // 1 source x0 pixels - movd xmm1, [esi + edx * 4] // 1 source x1 pixels - pextrw eax, xmm2, 5 // get x2 integer. - pextrw edx, xmm2, 7 // get x3 integer. - paddd xmm2, xmm3 // x += dx - punpckldq xmm0, xmm1 // x0 x1 - - movd xmm1, [esi + eax * 4] // 1 source x2 pixels - movd xmm4, [esi + edx * 4] // 1 source x3 pixels - pextrw eax, xmm2, 1 // get x0 integer. next iteration. - pextrw edx, xmm2, 3 // get x1 integer. next iteration. - punpckldq xmm1, xmm4 // x2 x3 - punpcklqdq xmm0, xmm1 // x0 x1 x2 x3 - movdqu [edi], xmm0 - lea edi, [edi + 16] - sub ecx, 4 // 4 pixels - jge xloop4 - - xloop49: - test ecx, 2 - je xloop29 - - // 2 Pixels. - movd xmm0, [esi + eax * 4] // 1 source x0 pixels - movd xmm1, [esi + edx * 4] // 1 source x1 pixels - pextrw eax, xmm2, 5 // get x2 integer. - punpckldq xmm0, xmm1 // x0 x1 - - movq qword ptr [edi], xmm0 - lea edi, [edi + 8] - - xloop29: - test ecx, 1 - je xloop99 - - // 1 Pixels. - movd xmm0, [esi + eax * 4] // 1 source x2 pixels - movd dword ptr [edi], xmm0 - xloop99: - - pop esi - pop edi - ret - } -} - -// Bilinear row filtering combines 2x1 -> 1x1. SSSE3 version. -// TODO(fbarchard): Port to Neon - -// Shuffle table for arranging 2 pixels into pairs for pmaddubsw -static const uvec8 kShuffleColARGB = { - 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u, // bbggrraa 1st pixel - 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u // bbggrraa 2nd pixel -}; - -// Shuffle table for duplicating 2 fractions into 8 bytes each -static const uvec8 kShuffleFractions = { - 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, -}; - -__declspec(naked) void ScaleARGBFilterCols_SSSE3(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx) { - __asm { - push esi - push edi - mov edi, [esp + 8 + 4] // dst_argb - mov esi, [esp + 8 + 8] // src_argb - mov ecx, [esp + 8 + 12] // dst_width - movd xmm2, [esp + 8 + 16] // x - movd xmm3, [esp + 8 + 20] // dx - movdqa xmm4, xmmword ptr kShuffleColARGB - movdqa xmm5, xmmword ptr kShuffleFractions - pcmpeqb xmm6, xmm6 // generate 0x007f for inverting fraction. - psrlw xmm6, 9 - pextrw eax, xmm2, 1 // get x0 integer. preroll - sub ecx, 2 - jl xloop29 - - movdqa xmm0, xmm2 // x1 = x0 + dx - paddd xmm0, xmm3 - punpckldq xmm2, xmm0 // x0 x1 - punpckldq xmm3, xmm3 // dx dx - paddd xmm3, xmm3 // dx * 2, dx * 2 - pextrw edx, xmm2, 3 // get x1 integer. preroll - - // 2 Pixel loop. - xloop2: - movdqa xmm1, xmm2 // x0, x1 fractions. - paddd xmm2, xmm3 // x += dx - movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels - psrlw xmm1, 9 // 7 bit fractions. - movhps xmm0, qword ptr [esi + edx * 4] // 2 source x1 pixels - pshufb xmm1, xmm5 // 0000000011111111 - pshufb xmm0, xmm4 // arrange pixels into pairs - pxor xmm1, xmm6 // 0..7f and 7f..0 - pmaddubsw xmm0, xmm1 // argb_argb 16 bit, 2 pixels. - pextrw eax, xmm2, 1 // get x0 integer. next iteration. - pextrw edx, xmm2, 3 // get x1 integer. next iteration. - psrlw xmm0, 7 // argb 8.7 fixed point to low 8 bits. - packuswb xmm0, xmm0 // argb_argb 8 bits, 2 pixels. - movq qword ptr [edi], xmm0 - lea edi, [edi + 8] - sub ecx, 2 // 2 pixels - jge xloop2 - - xloop29: - - add ecx, 2 - 1 - jl xloop99 - - // 1 pixel remainder - psrlw xmm2, 9 // 7 bit fractions. - movq xmm0, qword ptr [esi + eax * 4] // 2 source x0 pixels - pshufb xmm2, xmm5 // 00000000 - pshufb xmm0, xmm4 // arrange pixels into pairs - pxor xmm2, xmm6 // 0..7f and 7f..0 - pmaddubsw xmm0, xmm2 // argb 16 bit, 1 pixel. - psrlw xmm0, 7 - packuswb xmm0, xmm0 // argb 8 bits, 1 pixel. - movd [edi], xmm0 - - xloop99: - - pop edi - pop esi - ret - } -} - -// Reads 4 pixels, duplicates them and writes 8 pixels. -__declspec(naked) void ScaleARGBColsUp2_SSE2(uint8_t* dst_argb, - const uint8_t* src_argb, - int dst_width, - int x, - int dx) { - __asm { - mov edx, [esp + 4] // dst_argb - mov eax, [esp + 8] // src_argb - mov ecx, [esp + 12] // dst_width - - wloop: - movdqu xmm0, [eax] - lea eax, [eax + 16] - movdqa xmm1, xmm0 - punpckldq xmm0, xmm0 - punpckhdq xmm1, xmm1 - movdqu [edx], xmm0 - movdqu [edx + 16], xmm1 - lea edx, [edx + 32] - sub ecx, 8 - jg wloop - - ret - } -} - +#ifdef HAS_FIXEDDIV_X86 // Divide num by div and return as 16.16 fixed point result. __declspec(naked) int FixedDiv_X86(int num, int div) { __asm { @@ -1153,7 +261,8 @@ __declspec(naked) int FixedDiv1_X86(int num, int div) { ret } } -#endif // !defined(LIBYUV_DISABLE_X86) && defined(_M_IX86) +#endif // HAS_FIXEDDIV_X86 +#endif // !defined(LIBYUV_DISABLE_X86) && defined(_MSC_VER) && defined(_M_IX86) // The following are available for Visual C 32/64 bit, and for clang with // LIBYUV_ENABLE_ROWWIN: @@ -1164,145 +273,739 @@ __declspec(naked) int FixedDiv1_X86(int num, int div) { defined(LIBYUV_ENABLE_ROWWIN)) #if defined(__clang__) || defined(__GNUC__) -#define LIBYUV_TARGET_SSSE3 __attribute__((target("ssse3"))) #define LIBYUV_TARGET_AVX2 __attribute__((target("avx2"))) #define LIBYUV_TARGET_AVX512BW \ __attribute__((target("avx512bw,avx512vl,avx512f"))) #define LIBYUV_TARGET_AVX512VBMI \ __attribute__((target("avx512vbmi,avx512bw,avx512vl,avx512f"))) #else -#define LIBYUV_TARGET_SSSE3 #define LIBYUV_TARGET_AVX2 #define LIBYUV_TARGET_AVX512BW #define LIBYUV_TARGET_AVX512VBMI #endif // defined(__clang__) || defined(__GNUC__) -#ifdef HAS_SCALEROWDOWN38_SSSE3 +static inline uint16_t LoadU16(const void* p) { + uint16_t v; + memcpy(&v, p, sizeof(v)); + return v; +} + +static inline uint32_t LoadU32(const void* p) { + uint32_t v; + memcpy(&v, p, sizeof(v)); + return v; +} + +static inline uint64_t LoadU64(const void* p) { + uint64_t v; + memcpy(&v, p, sizeof(v)); + return v; +} + +static inline void StoreU32(void* p, uint32_t v) { + memcpy(p, &v, sizeof(v)); +} + +#if defined(HAS_FIXEDDIV_X86) && \ + !(defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__) && \ + !defined(LIBYUV_ENABLE_ROWWIN)) +int FixedDiv_X86(int num, int div) { + return (int)(((int64_t)num * 65536) / div); +} +#endif + +#if defined(HAS_FIXEDDIV1_X86) && \ + !(defined(_MSC_VER) && defined(_M_IX86) && !defined(__clang__) && \ + !defined(LIBYUV_ENABLE_ROWWIN)) +int FixedDiv1_X86(int num, int div) { + return (int)((((int64_t)num * 65536) - 0x00010001) / (div - 1)); +} +#endif + +#ifdef HAS_SCALEROWDOWN2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowDown2_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + __m256i s0 = _mm256_srli_epi16( + _mm256_loadu_si256((const __m256i*)src_ptr), 8); + __m256i s1 = _mm256_srli_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), 8); + src_ptr += 64; + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 32; + dst_width -= 32; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown2Linear_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m256i ones = _mm256_set1_epi8(1); + const __m256i zero = _mm256_setzero_si256(); + while (dst_width > 0) { + __m256i s0 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)src_ptr), ones); + __m256i s1 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones); + src_ptr += 64; + s0 = _mm256_avg_epu16(s0, zero); + s1 = _mm256_avg_epu16(s1, zero); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 32; + dst_width -= 32; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown2Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i ones = _mm256_set1_epi8(1); + const __m256i zero = _mm256_setzero_si256(); + while (dst_width > 0) { + __m256i s0 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)src_ptr), ones); + __m256i s1 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones); + __m256i s2 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), ones); + __m256i s3 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)), ones); + src_ptr += 64; + s0 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(s0, s2), 1), zero); + s1 = _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(s1, s3), 1), zero); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 32; + dst_width -= 32; + } +} +#endif // HAS_SCALEROWDOWN2_AVX2 + +#ifdef HAS_SCALEROWDOWN2_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowDown2_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + __m256i s0 = _mm256_srli_epi32( + _mm256_loadu_si256((const __m256i*)src_ptr), 16); + __m256i s1 = _mm256_srli_epi32( + _mm256_loadu_si256((const __m256i*)(src_ptr + 16)), 16); + src_ptr += 32; + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 16; + dst_width -= 16; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown2Linear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m256i mask = _mm256_set1_epi32(0x0000ffff); + while (dst_width > 0) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 16)); + src_ptr += 32; + __m256i e = _mm256_packus_epi32(_mm256_and_si256(r0, mask), + _mm256_and_si256(r1, mask)); + __m256i o = _mm256_packus_epi32(_mm256_srli_epi32(r0, 16), + _mm256_srli_epi32(r1, 16)); + __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu16(e, o), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 16; + dst_width -= 16; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown2Box_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + int dst_width) { + const __m256i mask = _mm256_set1_epi32(0x0000ffff); + const __m256i round2 = _mm256_set1_epi32(2); + while (dst_width > 0) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 16)); + __m256i r2 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)); + __m256i r3 = + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 16)); + src_ptr += 32; + __m256i s0 = _mm256_add_epi32( + _mm256_add_epi32(_mm256_and_si256(r0, mask), _mm256_srli_epi32(r0, 16)), + _mm256_add_epi32(_mm256_and_si256(r2, mask), + _mm256_srli_epi32(r2, 16))); + __m256i s1 = _mm256_add_epi32( + _mm256_add_epi32(_mm256_and_si256(r1, mask), _mm256_srli_epi32(r1, 16)), + _mm256_add_epi32(_mm256_and_si256(r3, mask), + _mm256_srli_epi32(r3, 16))); + s0 = _mm256_srli_epi32(_mm256_add_epi32(s0, round2), 2); + s1 = _mm256_srli_epi32(_mm256_add_epi32(s1, round2), 2); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWDOWN2_16_AVX2 + +#ifdef HAS_SCALEROWDOWN2_AVX512BW +LIBYUV_TARGET_AVX512BW +void ScaleRowDown2_AVX512BW(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + while (dst_width >= 32) { + __m512i s = + _mm512_srli_epi16(_mm512_loadu_si512((const __m512i*)src_ptr), 8); + src_ptr += 64; + _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s)); + dst_ptr += 32; + dst_width -= 32; + } + if (dst_width > 0) { + __mmask32 k = (__mmask32)((1ULL << dst_width) - 1); + __m512i s = _mm512_srli_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), 8); + _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s)); + } +} + +LIBYUV_TARGET_AVX512BW +void ScaleRowDown2Linear_AVX512BW(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m512i ones = _mm512_set1_epi8(1); + const __m512i zero = _mm512_setzero_si512(); + while (dst_width >= 32) { + __m512i s = _mm512_avg_epu16( + _mm512_maddubs_epi16(_mm512_loadu_si512((const __m512i*)src_ptr), ones), + zero); + src_ptr += 64; + _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s)); + dst_ptr += 32; + dst_width -= 32; + } + if (dst_width > 0) { + __mmask32 k = (__mmask32)((1ULL << dst_width) - 1); + __m512i s = _mm512_avg_epu16( + _mm512_maddubs_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), ones), + zero); + _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s)); + } +} + +LIBYUV_TARGET_AVX512BW +void ScaleRowDown2Box_AVX512BW(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m512i ones = _mm512_set1_epi8(1); + const __m512i zero = _mm512_setzero_si512(); + while (dst_width >= 32) { + __m512i s0 = _mm512_maddubs_epi16( + _mm512_loadu_si512((const __m512i*)src_ptr), ones); + __m512i s1 = _mm512_maddubs_epi16( + _mm512_loadu_si512((const __m512i*)(src_ptr + src_stride)), ones); + src_ptr += 64; + __m512i s = + _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(s0, s1), 1), zero); + _mm256_storeu_si256((__m256i*)dst_ptr, _mm512_cvtepi16_epi8(s)); + dst_ptr += 32; + dst_width -= 32; + } + if (dst_width > 0) { + __mmask32 k = (__mmask32)((1ULL << dst_width) - 1); + __m512i s0 = + _mm512_maddubs_epi16(_mm512_maskz_loadu_epi16(k, src_ptr), ones); + __m512i s1 = _mm512_maddubs_epi16( + _mm512_maskz_loadu_epi16(k, src_ptr + src_stride), ones); + __m512i s = + _mm512_avg_epu16(_mm512_srli_epi16(_mm512_add_epi16(s0, s1), 1), zero); + _mm256_mask_storeu_epi8(dst_ptr, k, _mm512_cvtepi16_epi8(s)); + } +} + +LIBYUV_TARGET_AVX512BW +void ScaleRowDown2Box_Odd_AVX512BW(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + dst_width -= 1; + if (dst_width > 0) { + ScaleRowDown2Box_AVX512BW(src_ptr, src_stride, dst_ptr, dst_width); + src_ptr += dst_width * 2; + dst_ptr += dst_width; + } + dst_ptr[0] = (uint8_t)((src_ptr[0] + src_ptr[src_stride] + 1) >> 1); +} +#endif // HAS_SCALEROWDOWN2_AVX512BW + +#ifdef HAS_SCALEROWDOWN4_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowDown4_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m256i mask = _mm256_set1_epi32(0x00ff0000); + while (dst_width > 0) { + __m256i s0 = + _mm256_and_si256(_mm256_loadu_si256((const __m256i*)src_ptr), mask); + __m256i s1 = _mm256_and_si256( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), mask); + src_ptr += 64; + __m256i t = _mm256_srli_epi16( + _mm256_permute4x64_epi64(_mm256_packus_epi16(s0, s1), 0xd8), 8); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(t, t), 0xd8); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d)); + dst_ptr += 16; + dst_width -= 16; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown4Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i ones = _mm256_set1_epi8(1); + const __m256i round8 = _mm256_set1_epi16(8); + const ptrdiff_t stridex3 = src_stride * 3; + while (dst_width > 0) { + __m256i s0 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)src_ptr), ones); + __m256i s1 = _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), ones); + s0 = _mm256_add_epi16( + s0, _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), + ones)); + s1 = _mm256_add_epi16( + s1, _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)), + ones)); + s0 = _mm256_add_epi16( + s0, _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2)), + ones)); + s1 = _mm256_add_epi16( + s1, + _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2 + 32)), + ones)); + s0 = _mm256_add_epi16( + s0, + _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + stridex3)), ones)); + s1 = _mm256_add_epi16( + s1, _mm256_maddubs_epi16( + _mm256_loadu_si256((const __m256i*)(src_ptr + stridex3 + 32)), + ones)); + src_ptr += 64; + __m256i t = _mm256_permute4x64_epi64(_mm256_hadd_epi16(s0, s1), 0xd8); + t = _mm256_srli_epi16(_mm256_add_epi16(t, round8), 4); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi16(t, t), 0xd8); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d)); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWDOWN4_AVX2 + +#ifdef HAS_SCALEROWDOWN34_AVX2 +static const uvec8 kShuf0 = {0, 1, 3, 4, 5, 7, 8, 9, + 128, 128, 128, 128, 128, 128, 128, 128}; +static const uvec8 kShuf1 = {3, 4, 5, 7, 8, 9, 11, 12, + 128, 128, 128, 128, 128, 128, 128, 128}; +static const uvec8 kShuf2 = {5, 7, 8, 9, 11, 12, 13, 15, + 128, 128, 128, 128, 128, 128, 128, 128}; +static const uvec8 kShuf01 = {0, 1, 1, 2, 2, 3, 4, 5, 5, 6, 6, 7, 8, 9, 9, 10}; +static const uvec8 kShuf11 = {2, 3, 4, 5, 5, 6, 6, 7, + 8, 9, 9, 10, 10, 11, 12, 13}; +static const uvec8 kShuf21 = {5, 6, 6, 7, 8, 9, 9, 10, + 10, 11, 12, 13, 13, 14, 14, 15}; +static const uvec8 kMadd01 = {3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2}; +static const uvec8 kMadd11 = {1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1}; +static const uvec8 kMadd21 = {2, 2, 1, 3, 3, 1, 2, 2, 1, 3, 3, 1, 2, 2, 1, 3}; + +LIBYUV_TARGET_AVX2 +void ScaleRowDown34_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + (void)src_stride; + const __m256i shuf0 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf0)); + const __m256i shuf1 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf1)); + const __m256i shuf2 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf2)); + while (dst_width >= 48) { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i s1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 32)); + src_ptr += 64; + __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20); + __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31); + __m256i mid = _mm256_alignr_epi8(hi, lo, 8); + __m256i d0 = _mm256_shuffle_epi8(lo, shuf0); + __m256i d1 = _mm256_shuffle_epi8(mid, shuf1); + __m256i d2 = _mm256_shuffle_epi8(hi, shuf2); + __m256i d01 = _mm256_unpacklo_epi64(d0, d1); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(d2)); + _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(d2, 1)); + dst_ptr += 48; + dst_width -= 48; + } + if (dst_width > 0) { + __m256i s0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m128i lo = _mm256_castsi256_si128(s0); + __m128i hi = _mm256_extracti128_si256(s0, 1); + __m128i mid = _mm_alignr_epi8(hi, lo, 8); + __m128i d0 = _mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf0)); + __m128i d1 = _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf1)); + __m128i d2 = _mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf2)); + _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(d0, d1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), d2); + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown34_1_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i shuf01 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf01)); + const __m256i shuf11 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf11)); + const __m256i shuf21 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf21)); + const __m256i madd01 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd01)); + const __m256i madd11 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd11)); + const __m256i madd21 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd21)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width >= 48) { + __m256i s0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_ptr), + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride))); + __m256i s1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32))); + src_ptr += 64; + __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20); + __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31); + __m256i mid = _mm256_alignr_epi8(hi, lo, 8); + __m256i p0 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(lo, shuf01), madd01), + round2), + 2); + __m256i p1 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(mid, shuf11), madd11), + round2), + 2); + __m256i p2 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(hi, shuf21), madd21), + round2), + 2); + p0 = _mm256_packus_epi16(p0, p0); + p1 = _mm256_packus_epi16(p1, p1); + p2 = _mm256_packus_epi16(p2, p2); + __m256i d01 = _mm256_unpacklo_epi64(p0, p1); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(p2)); + _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(p2, 1)); + dst_ptr += 48; + dst_width -= 48; + } + if (dst_width > 0) { + __m256i s0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_ptr), + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride))); + __m128i lo = _mm256_castsi256_si128(s0); + __m128i hi = _mm256_extracti128_si256(s0, 1); + __m128i mid = _mm_alignr_epi8(hi, lo, 8); + __m128i r2 = _mm256_castsi256_si128(round2); + __m128i p0 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16(_mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf01)), + _mm256_castsi256_si128(madd01)), + r2), + 2); + __m128i p1 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16( + _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf11)), + _mm256_castsi256_si128(madd11)), + r2), + 2); + __m128i p2 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16(_mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf21)), + _mm256_castsi256_si128(madd21)), + r2), + 2); + p0 = _mm_packus_epi16(p0, p0); + p1 = _mm_packus_epi16(p1, p1); + p2 = _mm_packus_epi16(p2, p2); + _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(p0, p1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), p2); + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown34_0_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i shuf01 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf01)); + const __m256i shuf11 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf11)); + const __m256i shuf21 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf21)); + const __m256i madd01 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd01)); + const __m256i madd11 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd11)); + const __m256i madd21 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kMadd21)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width >= 48) { + __m256i a0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i b0 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)); + __m256i s0 = _mm256_avg_epu8(a0, _mm256_avg_epu8(a0, b0)); + __m256i a1 = _mm256_loadu_si256((const __m256i*)(src_ptr + 32)); + __m256i b1 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride + 32)); + __m256i s1 = _mm256_avg_epu8(a1, _mm256_avg_epu8(a1, b1)); + src_ptr += 64; + __m256i lo = _mm256_permute2x128_si256(s0, s1, 0x20); + __m256i hi = _mm256_permute2x128_si256(s0, s1, 0x31); + __m256i mid = _mm256_alignr_epi8(hi, lo, 8); + __m256i p0 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(lo, shuf01), madd01), + round2), + 2); + __m256i p1 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(mid, shuf11), madd11), + round2), + 2); + __m256i p2 = _mm256_srli_epi16( + _mm256_adds_epi16( + _mm256_maddubs_epi16(_mm256_shuffle_epi8(hi, shuf21), madd21), + round2), + 2); + p0 = _mm256_packus_epi16(p0, p0); + p1 = _mm256_packus_epi16(p1, p1); + p2 = _mm256_packus_epi16(p2, p2); + __m256i d01 = _mm256_unpacklo_epi64(p0, p1); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d01)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), _mm256_castsi256_si128(p2)); + _mm_storeu_si128((__m128i*)(dst_ptr + 24), _mm256_extracti128_si256(d01, 1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 40), _mm256_extracti128_si256(p2, 1)); + dst_ptr += 48; + dst_width -= 48; + } + if (dst_width > 0) { + __m256i a0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i b0 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)); + __m256i s0 = _mm256_avg_epu8(a0, _mm256_avg_epu8(a0, b0)); + __m128i lo = _mm256_castsi256_si128(s0); + __m128i hi = _mm256_extracti128_si256(s0, 1); + __m128i mid = _mm_alignr_epi8(hi, lo, 8); + __m128i r2 = _mm256_castsi256_si128(round2); + __m128i p0 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16(_mm_shuffle_epi8(lo, _mm256_castsi256_si128(shuf01)), + _mm256_castsi256_si128(madd01)), + r2), + 2); + __m128i p1 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16( + _mm_shuffle_epi8(mid, _mm256_castsi256_si128(shuf11)), + _mm256_castsi256_si128(madd11)), + r2), + 2); + __m128i p2 = _mm_srli_epi16( + _mm_adds_epi16( + _mm_maddubs_epi16(_mm_shuffle_epi8(hi, _mm256_castsi256_si128(shuf21)), + _mm256_castsi256_si128(madd21)), + r2), + 2); + p0 = _mm_packus_epi16(p0, p0); + p1 = _mm_packus_epi16(p1, p1); + p2 = _mm_packus_epi16(p2, p2); + _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi64(p0, p1)); + _mm_storel_epi64((__m128i*)(dst_ptr + 16), p2); + } +} +#endif // HAS_SCALEROWDOWN34_AVX2 + +#ifdef HAS_SCALEROWDOWN38_AVX2 static const uvec8 kShuf38a = {0, 3, 6, 8, 11, 14, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; - static const uvec8 kShuf38b = {128, 128, 128, 128, 128, 128, 0, 3, 6, 8, 11, 14, 128, 128, 128, 128}; - -// Arrange words 0,3,6 into 0,1,2 static const uvec8 kShufAc = {0, 1, 6, 7, 12, 13, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128}; - -// Arrange words 0,3,6 into 3,4,5 static const uvec8 kShufAc3 = {128, 128, 128, 128, 128, 128, 0, 1, 6, 7, 12, 13, 128, 128, 128, 128}; - -// Scaling values for boxes of 3x3 and 2x3 static const uvec16 kScaleAc33 = {65536 / 9, 65536 / 9, 65536 / 6, 65536 / 9, 65536 / 9, 65536 / 6, 0, 0}; - -// Arrange first value for pixels 0,1,2,3,4,5 static const uvec8 kShufAb0 = {0, 128, 3, 128, 6, 128, 8, 128, 11, 128, 14, 128, 128, 128, 128, 128}; - -// Arrange second value for pixels 0,1,2,3,4,5 static const uvec8 kShufAb1 = {1, 128, 4, 128, 7, 128, 9, 128, 12, 128, 15, 128, 128, 128, 128, 128}; - -// Arrange third value for pixels 0,1,2,3,4,5 static const uvec8 kShufAb2 = {2, 128, 5, 128, 128, 128, 10, 128, 13, 128, 128, 128, 128, 128, 128, 128}; - -// Scaling values for boxes of 3x2 and 2x2 static const uvec16 kScaleAb2 = {65536 / 3, 65536 / 3, 65536 / 2, 65536 / 3, 65536 / 3, 65536 / 2, 0, 0}; -// Scale 32 pixels to 12 -LIBYUV_TARGET_SSSE3 -void ScaleRowDown38_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { +LIBYUV_TARGET_AVX2 +void ScaleRowDown38_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { (void)src_stride; - const __m128i shuf38a = _mm_loadu_si128((const __m128i*)&kShuf38a); - const __m128i shuf38b = _mm_loadu_si128((const __m128i*)&kShuf38b); + const __m256i shuf38 = _mm256_inserti128_si256( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)&kShuf38a)), + _mm_loadu_si128((const __m128i*)&kShuf38b), 1); + while (dst_width >= 24) { + __m256i s0 = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), shuf38); + __m256i s1 = _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ptr + 32)), shuf38); + src_ptr += 64; + __m128i d0 = + _mm_or_si128(_mm256_castsi256_si128(s0), _mm256_extracti128_si256(s0, 1)); + __m128i d1 = + _mm_or_si128(_mm256_castsi256_si128(s1), _mm256_extracti128_si256(s1, 1)); + _mm_storel_epi64((__m128i*)dst_ptr, d0); + StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d0, 8))); + _mm_storel_epi64((__m128i*)(dst_ptr + 12), d1); + StoreU32(dst_ptr + 20, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d1, 8))); + dst_ptr += 24; + dst_width -= 24; + } + if (dst_width > 0) { + __m256i s0 = + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), shuf38); + __m128i d0 = + _mm_or_si128(_mm256_castsi256_si128(s0), _mm256_extracti128_si256(s0, 1)); + _mm_storel_epi64((__m128i*)dst_ptr, d0); + StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d0, 8))); + } +} + +LIBYUV_TARGET_AVX2 +void ScaleRowDown38_2_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i shuf_ab0 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb0)); + const __m256i shuf_ab1 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb1)); + const __m256i shuf_ab2 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAb2)); + const __m256i scale_ab2 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kScaleAb2)); + const __m256i zero = _mm256_setzero_si256(); while (dst_width > 0) { - __m128i s0 = _mm_loadu_si128((const __m128i*)src_ptr); - __m128i s1 = _mm_loadu_si128((const __m128i*)(src_ptr + 16)); + __m256i r = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_ptr), + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride))); src_ptr += 32; - s0 = _mm_shuffle_epi8(s0, shuf38a); - s1 = _mm_shuffle_epi8(s1, shuf38b); - s0 = _mm_adds_epu8(s0, s1); - _mm_storel_epi64((__m128i*)dst_ptr, s0); - *(uint32_t*)(dst_ptr + 8) = - (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(s0, 8)); + __m256i t = _mm256_adds_epu16( + _mm256_adds_epu16(_mm256_shuffle_epi8(r, shuf_ab0), + _mm256_shuffle_epi8(r, shuf_ab1)), + _mm256_shuffle_epi8(r, shuf_ab2)); + t = _mm256_packus_epi16(_mm256_mulhi_epu16(t, scale_ab2), zero); + __m128i d = _mm_or_si128(_mm256_castsi256_si128(t), + _mm_slli_si128(_mm256_extracti128_si256(t, 1), 6)); + _mm_storel_epi64((__m128i*)dst_ptr, d); + StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d, 8))); dst_ptr += 12; dst_width -= 12; } } -// Scale 16x3 pixels to 6x1 with interpolation -LIBYUV_TARGET_SSSE3 -void ScaleRowDown38_3_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - const __m128i shuf_ac = _mm_loadu_si128((const __m128i*)&kShufAc); - const __m128i shuf_ac3 = _mm_loadu_si128((const __m128i*)&kShufAc3); - const __m128i scale_ac33 = _mm_loadu_si128((const __m128i*)&kScaleAc33); - const __m128i zero = _mm_setzero_si128(); +LIBYUV_TARGET_AVX2 +void ScaleRowDown38_3_Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i shuf_ac = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAc)); + const __m256i shuf_ac3 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kShufAc3)); + const __m256i scale_ac33 = + _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i*)&kScaleAc33)); + const __m256i zero = _mm256_setzero_si256(); while (dst_width > 0) { - __m128i r0 = _mm_loadu_si128((const __m128i*)src_ptr); - __m128i r1 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride)); - __m128i r2 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride * 2)); - src_ptr += 16; - __m128i s0 = _mm_adds_epu16(_mm_adds_epu16(_mm_unpacklo_epi8(r0, zero), - _mm_unpacklo_epi8(r1, zero)), - _mm_unpacklo_epi8(r2, zero)); - __m128i s1 = _mm_adds_epu16(_mm_adds_epu16(_mm_unpackhi_epi8(r0, zero), - _mm_unpackhi_epi8(r1, zero)), - _mm_unpackhi_epi8(r2, zero)); - __m128i t0 = _mm_adds_epu16(_mm_adds_epu16(s0, _mm_srli_si128(s0, 2)), - _mm_srli_si128(s0, 4)); - __m128i t1 = _mm_adds_epu16(_mm_adds_epu16(s1, _mm_srli_si128(s1, 2)), - _mm_srli_si128(s1, 4)); - t0 = _mm_shuffle_epi8(t0, shuf_ac); - t1 = _mm_shuffle_epi8(t1, shuf_ac3); - t0 = _mm_adds_epu16(t0, t1); - t0 = _mm_mulhi_epu16(t0, scale_ac33); - t0 = _mm_packus_epi16(t0, t0); - *(uint32_t*)dst_ptr = (uint32_t)_mm_cvtsi128_si32(t0); - *(uint16_t*)(dst_ptr + 4) = - (uint16_t)_mm_cvtsi128_si32(_mm_srli_si128(t0, 4)); - dst_ptr += 6; - dst_width -= 6; + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_ptr); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)); + __m256i r2 = + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride * 2)); + src_ptr += 32; + __m256i s0 = + _mm256_adds_epu16(_mm256_adds_epu16(_mm256_unpacklo_epi8(r0, zero), + _mm256_unpacklo_epi8(r1, zero)), + _mm256_unpacklo_epi8(r2, zero)); + __m256i s1 = + _mm256_adds_epu16(_mm256_adds_epu16(_mm256_unpackhi_epi8(r0, zero), + _mm256_unpackhi_epi8(r1, zero)), + _mm256_unpackhi_epi8(r2, zero)); + __m256i t0 = _mm256_shuffle_epi8( + _mm256_adds_epu16(_mm256_adds_epu16(s0, _mm256_srli_si256(s0, 2)), + _mm256_srli_si256(s0, 4)), + shuf_ac); + __m256i t1 = _mm256_shuffle_epi8( + _mm256_adds_epu16(_mm256_adds_epu16(s1, _mm256_srli_si256(s1, 2)), + _mm256_srli_si256(s1, 4)), + shuf_ac3); + __m256i t = _mm256_packus_epi16( + _mm256_mulhi_epu16(_mm256_adds_epu16(t0, t1), scale_ac33), zero); + __m128i d = _mm_or_si128(_mm256_castsi256_si128(t), + _mm_slli_si128(_mm256_extracti128_si256(t, 1), 6)); + _mm_storel_epi64((__m128i*)dst_ptr, d); + StoreU32(dst_ptr + 8, (uint32_t)_mm_cvtsi128_si32(_mm_srli_si128(d, 8))); + dst_ptr += 12; + dst_width -= 12; } } - -// Scale 16x2 pixels to 6x1 with interpolation -LIBYUV_TARGET_SSSE3 -void ScaleRowDown38_2_Box_SSSE3(const uint8_t* src_ptr, - ptrdiff_t src_stride, - uint8_t* dst_ptr, - int dst_width) { - const __m128i shuf_ab0 = _mm_loadu_si128((const __m128i*)&kShufAb0); - const __m128i shuf_ab1 = _mm_loadu_si128((const __m128i*)&kShufAb1); - const __m128i shuf_ab2 = _mm_loadu_si128((const __m128i*)&kShufAb2); - const __m128i scale_ab2 = _mm_loadu_si128((const __m128i*)&kScaleAb2); - while (dst_width > 0) { - __m128i r0 = _mm_loadu_si128((const __m128i*)src_ptr); - __m128i r1 = _mm_loadu_si128((const __m128i*)(src_ptr + src_stride)); - src_ptr += 16; - r0 = _mm_avg_epu8(r0, r1); - __m128i t0 = _mm_shuffle_epi8(r0, shuf_ab0); - __m128i t1 = _mm_shuffle_epi8(r0, shuf_ab1); - __m128i t2 = _mm_shuffle_epi8(r0, shuf_ab2); - t0 = _mm_adds_epu16(_mm_adds_epu16(t0, t1), t2); - t0 = _mm_mulhi_epu16(t0, scale_ab2); - t0 = _mm_packus_epi16(t0, t0); - *(uint32_t*)dst_ptr = (uint32_t)_mm_cvtsi128_si32(t0); - *(uint16_t*)(dst_ptr + 4) = - (uint16_t)_mm_cvtsi128_si32(_mm_srli_si128(t0, 4)); - dst_ptr += 6; - dst_width -= 6; - } -} -#endif // HAS_SCALEROWDOWN38_SSSE3 +#endif // HAS_SCALEROWDOWN38_AVX2 #ifdef HAS_SCALEADDROW_SSE2 // Reads 16 bytes and accumulates to 16 shorts at a time. @@ -1390,10 +1093,12 @@ void ScaleAddRow_AVX512BW(const uint8_t* src_ptr, } #endif // HAS_SCALEADDROW_AVX512BW -#ifdef HAS_SCALEADDCOLS_AVX512BW +#if defined(HAS_SCALEADDCOLS1_AVX512BW) || defined(HAS_SCALEADDCOLS2_AVX512BW) static const uint8_t kSequence0_15[16] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; +#endif +#ifdef HAS_SCALEADDCOLS1_AVX512BW LIBYUV_TARGET_AVX512BW void ScaleAddCols1_AVX512BW(int dst_width, int boxheight, @@ -1543,7 +1248,9 @@ void ScaleAddCols1_AVX512BW(int dst_width, } } } +#endif // HAS_SCALEADDCOLS1_AVX512BW +#ifdef HAS_SCALEADDCOLS2_AVX512BW LIBYUV_TARGET_AVX512BW void ScaleAddCols2_AVX512BW(int dst_width, int boxheight, @@ -1745,7 +1452,7 @@ void ScaleAddCols2_AVX512BW(int dst_width, } } } -#endif // HAS_SCALEADDCOLS_AVX512BW +#endif // HAS_SCALEADDCOLS2_AVX512BW #ifdef HAS_SCALEUVADDCOLS3_AVX2 static const ulvec32 kPermUV3_0 = {0, 3, 6, 1, 4, 7, 2, 5}; @@ -1926,7 +1633,956 @@ void ScaleUVAddCols3_AVX512VBMI(int dst_width, } #endif // HAS_SCALEUVADDCOLS3_AVX512VBMI -#undef LIBYUV_TARGET_SSSE3 +#if defined(HAS_SCALEROWUP2_LINEAR_AVX2) || \ + defined(HAS_SCALEROWUP2_BILINEAR_AVX2) +static const uvec8 kLinearMadd31 = {3, 1, 1, 3, 3, 1, 1, 3, + 3, 1, 1, 3, 3, 1, 1, 3}; +#endif + +#if defined(HAS_SCALEROWUP2_LINEAR_12_AVX2) || \ + defined(HAS_SCALEROWUP2_BILINEAR_12_AVX2) +static const uvec8 kLinearShuffleFar = {2, 3, 0, 1, 6, 7, 4, 5, + 10, 11, 8, 9, 14, 15, 12, 13}; +#endif + +#ifdef HAS_SCALEROWUP2_LINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Linear_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width) { + const __m256i madd31 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kLinearMadd31)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width > 0) { + __m256i s0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i s1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))), + 0xd8); + src_ptr += 16; + s0 = _mm256_unpacklo_epi16(s0, s0); + s1 = _mm256_unpacklo_epi16(s1, s1); + __m256i lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(s0, s1), madd31); + __m256i hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(s0, s1), madd31); + lo = _mm256_srli_epi16(_mm256_add_epi16(lo, round2), 2); + hi = _mm256_srli_epi16(_mm256_add_epi16(hi, round2), 2); + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(lo, hi)); + dst_ptr += 32; + dst_width -= 32; + } +} +#endif // HAS_SCALEROWUP2_LINEAR_AVX2 + +#ifdef HAS_SCALEROWUP2_BILINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Bilinear_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i madd31 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kLinearMadd31)); + const __m256i round8 = _mm256_set1_epi16(8); + while (dst_width > 0) { + __m256i a0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i a1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))), + 0xd8); + a0 = _mm256_unpacklo_epi16(a0, a0); + a1 = _mm256_unpacklo_epi16(a1, a1); + __m256i r0_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(a0, a1), madd31); + __m256i r0_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(a0, a1), madd31); + + __m256i b0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))), + 0xd8); + __m256i b1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))), + 0xd8); + src_ptr += 16; + b0 = _mm256_unpacklo_epi16(b0, b0); + b1 = _mm256_unpacklo_epi16(b1, b1); + __m256i r1_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(b0, b1), madd31); + __m256i r1_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(b0, b1), madd31); + + __m256i d0_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_lo, r0_lo), r0_lo), + _mm256_add_epi16(r1_lo, round8)), + 4); + __m256i d1_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_lo, r1_lo), r1_lo), + _mm256_add_epi16(r0_lo, round8)), + 4); + __m256i d0_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_hi, r0_hi), r0_hi), + _mm256_add_epi16(r1_hi, round8)), + 4); + __m256i d1_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_hi, r1_hi), r1_hi), + _mm256_add_epi16(r0_hi, round8)), + 4); + + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(d0_lo, d0_hi)); + _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), + _mm256_packus_epi16(d1_lo, d1_hi)); + dst_ptr += 32; + dst_width -= 32; + } +} +#endif // HAS_SCALEROWUP2_BILINEAR_AVX2 + +#ifdef HAS_SCALEROWUP2_LINEAR_12_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Linear_12_AVX2(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int dst_width) { + const __m256i shuf_far = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kLinearShuffleFar)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width > 0) { + __m256i s0 = + _mm256_permute4x64_epi64(_mm256_loadu_si256((const __m256i*)src_ptr), + 0xd8); + __m256i s1 = _mm256_permute4x64_epi64( + _mm256_loadu_si256((const __m256i*)(src_ptr + 1)), 0xd8); + src_ptr += 16; + __m256i near_lo = _mm256_unpacklo_epi16(s0, s1); + __m256i near_hi = _mm256_unpackhi_epi16(s0, s1); + __m256i far_lo = _mm256_shuffle_epi8(near_lo, shuf_far); + __m256i far_hi = _mm256_shuffle_epi8(near_hi, shuf_far); + __m256i d_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(near_lo, near_lo), + _mm256_add_epi16(near_lo, _mm256_add_epi16(far_lo, round2))), + 2); + __m256i d_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(near_hi, near_hi), + _mm256_add_epi16(near_hi, _mm256_add_epi16(far_hi, round2))), + 2); + _mm256_storeu_si256((__m256i*)dst_ptr, d_lo); + _mm256_storeu_si256((__m256i*)(dst_ptr + 16), d_hi); + dst_ptr += 32; + dst_width -= 32; + } +} +#endif // HAS_SCALEROWUP2_LINEAR_12_AVX2 + +#ifdef HAS_SCALEROWUP2_BILINEAR_12_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Bilinear_12_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i shuf_far = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kLinearShuffleFar)); + const __m256i round8 = _mm256_set1_epi16(8); + while (dst_width > 0) { + __m256i a0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i a1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 1))), + 0xd8); + __m256i near0 = _mm256_unpacklo_epi16(a0, a1); + __m256i far0 = _mm256_shuffle_epi8(near0, shuf_far); + __m256i r0 = _mm256_add_epi16(_mm256_add_epi16(near0, near0), + _mm256_add_epi16(near0, far0)); + + __m256i b0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))), + 0xd8); + __m256i b1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))), + 0xd8); + src_ptr += 8; + __m256i near1 = _mm256_unpacklo_epi16(b0, b1); + __m256i far1 = _mm256_shuffle_epi8(near1, shuf_far); + __m256i r1 = _mm256_add_epi16(_mm256_add_epi16(near1, near1), + _mm256_add_epi16(near1, far1)); + + __m256i d0 = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0, r0), r0), + _mm256_add_epi16(r1, round8)), + 4); + __m256i d1 = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1, r1), r1), + _mm256_add_epi16(r0, round8)), + 4); + _mm256_storeu_si256((__m256i*)dst_ptr, d0); + _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), d1); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWUP2_BILINEAR_12_AVX2 + +#ifdef HAS_SCALEROWUP2_LINEAR_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Linear_16_AVX2(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int dst_width) { + const __m256i round2 = _mm256_set1_epi32(2); + while (dst_width > 0) { + __m256i near_lo = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i near_hi = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 1))); + src_ptr += 8; + __m256i far_lo = _mm256_shuffle_epi32(near_lo, 0xb1); + __m256i far_hi = _mm256_shuffle_epi32(near_hi, 0xb1); + __m256i d_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(near_lo, near_lo), + _mm256_add_epi32(near_lo, _mm256_add_epi32(far_lo, round2))), + 2); + __m256i d_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(near_hi, near_hi), + _mm256_add_epi32(near_hi, _mm256_add_epi32(far_hi, round2))), + 2); + __m256i d = _mm256_shuffle_epi32(_mm256_packus_epi32(d_lo, d_hi), 0xd8); + _mm256_storeu_si256((__m256i*)dst_ptr, d); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWUP2_LINEAR_16_AVX2 + +#ifdef HAS_SCALEROWUP2_BILINEAR_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i round8 = _mm256_set1_epi32(8); + while (dst_width > 0) { + __m256i a0 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i a1 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 1))); + __m256i r0_lo = _mm256_add_epi32( + _mm256_add_epi32(a0, a0), + _mm256_add_epi32(a0, _mm256_shuffle_epi32(a0, 0xb1))); + __m256i r0_hi = _mm256_add_epi32( + _mm256_add_epi32(a1, a1), + _mm256_add_epi32(a1, _mm256_shuffle_epi32(a1, 0xb1))); + + __m256i b0 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))); + __m256i b1 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 1))); + src_ptr += 8; + __m256i r1_lo = _mm256_add_epi32( + _mm256_add_epi32(b0, b0), + _mm256_add_epi32(b0, _mm256_shuffle_epi32(b0, 0xb1))); + __m256i r1_hi = _mm256_add_epi32( + _mm256_add_epi32(b1, b1), + _mm256_add_epi32(b1, _mm256_shuffle_epi32(b1, 0xb1))); + + __m256i d0_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_lo, r0_lo), r0_lo), + _mm256_add_epi32(r1_lo, round8)), + 4); + __m256i d1_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_lo, r1_lo), r1_lo), + _mm256_add_epi32(r0_lo, round8)), + 4); + __m256i d0_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_hi, r0_hi), r0_hi), + _mm256_add_epi32(r1_hi, round8)), + 4); + __m256i d1_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_hi, r1_hi), r1_hi), + _mm256_add_epi32(r0_hi, round8)), + 4); + + _mm256_storeu_si256( + (__m256i*)dst_ptr, + _mm256_shuffle_epi32(_mm256_packus_epi32(d0_lo, d0_hi), 0xd8)); + _mm256_storeu_si256( + (__m256i*)(dst_ptr + dst_stride), + _mm256_shuffle_epi32(_mm256_packus_epi32(d1_lo, d1_hi), 0xd8)); + dst_ptr += 16; + dst_width -= 16; + } +} +#endif // HAS_SCALEROWUP2_BILINEAR_16_AVX2 + +#ifdef HAS_SCALEFILTERCOLS_AVX2 +static const int32_t kStep0_7_Filter[8] = {0, 1, 2, 3, 4, 5, 6, 7}; +static const uvec8 kShufFracCols = {0, 0, 4, 4, 8, 8, 12, 12, + 128, 128, 128, 128, 128, 128, 128, 128}; + +LIBYUV_TARGET_AVX2 +void ScaleFilterCols_AVX2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx) { + int n = dst_width & ~7; + int r = dst_width & 7; + if (n > 0) { + SIMD_ALIGNED(int32_t xi_buf[8]); + SIMD_ALIGNED(uint16_t pix_buf[8]); + __m256i vx = _mm256_add_epi32( + _mm256_set1_epi32(x), + _mm256_mullo_epi32( + _mm256_set1_epi32(dx), + _mm256_loadu_si256((const __m256i*)kStep0_7_Filter))); + const __m256i vdx8 = _mm256_slli_epi32(_mm256_set1_epi32(dx), 3); + const __m256i shuf_frac = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kShufFracCols)); + const __m128i mask_7f = _mm_set1_epi16(0x007f); + const __m128i mask_ff = _mm_set1_epi16(0x00ff); + const __m128i round_40 = _mm_set1_epi16(0x0040); + for (int i = 0; i < n; i += 8) { + _mm256_store_si256((__m256i*)xi_buf, _mm256_srai_epi32(vx, 16)); + __m256i f256 = _mm256_permute4x64_epi64( + _mm256_shuffle_epi8(_mm256_srli_epi16(vx, 9), shuf_frac), 0xd8); + vx = _mm256_add_epi32(vx, vdx8); + __m128i weights = _mm_xor_si128(_mm256_castsi256_si128(f256), mask_7f); + pix_buf[0] = LoadU16(src_ptr + xi_buf[0]); + pix_buf[1] = LoadU16(src_ptr + xi_buf[1]); + pix_buf[2] = LoadU16(src_ptr + xi_buf[2]); + pix_buf[3] = LoadU16(src_ptr + xi_buf[3]); + pix_buf[4] = LoadU16(src_ptr + xi_buf[4]); + pix_buf[5] = LoadU16(src_ptr + xi_buf[5]); + pix_buf[6] = LoadU16(src_ptr + xi_buf[6]); + pix_buf[7] = LoadU16(src_ptr + xi_buf[7]); + __m128i pix = _mm_load_si128((const __m128i*)pix_buf); + __m128i a = _mm_and_si128(pix, mask_ff); + __m128i res = _mm_srli_epi16( + _mm_add_epi16(_mm_add_epi16(_mm_maddubs_epi16(pix, weights), a), + round_40), + 7); + _mm_storel_epi64((__m128i*)dst_ptr, _mm_packus_epi16(res, res)); + dst_ptr += 8; + } + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + int xi = x >> 16; + int a = src_ptr[xi]; + int b = src_ptr[xi + 1]; + dst_ptr[j] = (uint8_t)(a + ((((x & 0xffff) >> 9) * (b - a) + 0x40) >> 7)); + x = (int)((uint32_t)x + (uint32_t)dx); + } +} +#endif // HAS_SCALEFILTERCOLS_AVX2 + +#ifdef HAS_SCALECOLSUP2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleColsUp2_AVX2(uint8_t* dst_ptr, + const uint8_t* src_ptr, + int dst_width, + int x, + int dx) { + (void)x; + (void)dx; + while (dst_width >= 32) { + __m256i s = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + src_ptr += 16; + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_unpacklo_epi8(s, s)); + dst_ptr += 32; + dst_width -= 32; + } + if (dst_width & 16) { + __m128i s = _mm_loadl_epi64((const __m128i*)src_ptr); + src_ptr += 8; + _mm_storeu_si128((__m128i*)dst_ptr, _mm_unpacklo_epi8(s, s)); + dst_ptr += 16; + } + if (dst_width & 8) { + __m128i s = _mm_cvtsi32_si128((int)LoadU32(src_ptr)); + _mm_storel_epi64((__m128i*)dst_ptr, _mm_unpacklo_epi8(s, s)); + } +} +#endif // HAS_SCALECOLSUP2_AVX2 + +#ifdef HAS_SCALEARGBROWDOWN2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDown2_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + __m256 s0 = _mm256_loadu_ps((const float*)src_argb); + __m256 s1 = _mm256_loadu_ps((const float*)(src_argb + 32)); + src_argb += 64; + __m256i d = _mm256_permute4x64_epi64( + _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0xdd)), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, d); + dst_argb += 32; + dst_width -= 8; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDown2Linear_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + while (dst_width > 0) { + __m256 s0 = _mm256_loadu_ps((const float*)src_argb); + __m256 s1 = _mm256_loadu_ps((const float*)(src_argb + 32)); + src_argb += 64; + __m256i e = _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0x88)); + __m256i o = _mm256_castps_si256(_mm256_shuffle_ps(s0, s1, 0xdd)); + __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, d); + dst_argb += 32; + dst_width -= 8; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDown2Box_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + uint8_t* dst_argb, + int dst_width) { + while (dst_width > 0) { + __m256i r0 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)src_argb), + _mm256_loadu_si256((const __m256i*)(src_argb + src_stride))); + __m256i r1 = _mm256_avg_epu8( + _mm256_loadu_si256((const __m256i*)(src_argb + 32)), + _mm256_loadu_si256((const __m256i*)(src_argb + src_stride + 32))); + src_argb += 64; + __m256i e = _mm256_castps_si256( + _mm256_shuffle_ps(_mm256_castsi256_ps(r0), _mm256_castsi256_ps(r1), + 0x88)); + __m256i o = _mm256_castps_si256( + _mm256_shuffle_ps(_mm256_castsi256_ps(r0), _mm256_castsi256_ps(r1), + 0xdd)); + __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, d); + dst_argb += 32; + dst_width -= 8; + } +} +#endif // HAS_SCALEARGBROWDOWN2_AVX2 + +#ifdef HAS_SCALEARGBROWDOWNEVEN_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDownEven_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width) { + (void)src_stride; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + SIMD_ALIGNED(uint32_t pix_buf[8]); + while (dst_width > 0) { + pix_buf[0] = LoadU32(src_argb); + pix_buf[1] = LoadU32(src_argb + step); + pix_buf[2] = LoadU32(src_argb + step * 2); + pix_buf[3] = LoadU32(src_argb + step * 3); + pix_buf[4] = LoadU32(src_argb + step * 4); + pix_buf[5] = LoadU32(src_argb + step * 5); + pix_buf[6] = LoadU32(src_argb + step * 6); + pix_buf[7] = LoadU32(src_argb + step * 7); + src_argb += step * 8; + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_load_si256((const __m256i*)pix_buf)); + dst_argb += 32; + dst_width -= 8; + } +} + +LIBYUV_TARGET_AVX2 +void ScaleARGBRowDownEvenBox_AVX2(const uint8_t* src_argb, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_argb, + int dst_width) { + const uint8_t* row1 = src_argb + src_stride; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + SIMD_ALIGNED(uint64_t r0_lo[4]); + SIMD_ALIGNED(uint64_t r0_hi[4]); + SIMD_ALIGNED(uint64_t r1_lo[4]); + SIMD_ALIGNED(uint64_t r1_hi[4]); + while (dst_width > 0) { + r0_lo[0] = LoadU64(src_argb); + r0_lo[1] = LoadU64(src_argb + step); + r0_lo[2] = LoadU64(src_argb + step * 2); + r0_lo[3] = LoadU64(src_argb + step * 3); + r0_hi[0] = LoadU64(src_argb + step * 4); + r0_hi[1] = LoadU64(src_argb + step * 5); + r0_hi[2] = LoadU64(src_argb + step * 6); + r0_hi[3] = LoadU64(src_argb + step * 7); + r1_lo[0] = LoadU64(row1); + r1_lo[1] = LoadU64(row1 + step); + r1_lo[2] = LoadU64(row1 + step * 2); + r1_lo[3] = LoadU64(row1 + step * 3); + r1_hi[0] = LoadU64(row1 + step * 4); + r1_hi[1] = LoadU64(row1 + step * 5); + r1_hi[2] = LoadU64(row1 + step * 6); + r1_hi[3] = LoadU64(row1 + step * 7); + src_argb += step * 8; + row1 += step * 8; + __m256i a0 = _mm256_avg_epu8(_mm256_load_si256((const __m256i*)r0_lo), + _mm256_load_si256((const __m256i*)r1_lo)); + __m256i a1 = _mm256_avg_epu8(_mm256_load_si256((const __m256i*)r0_hi), + _mm256_load_si256((const __m256i*)r1_hi)); + __m256i e = _mm256_castps_si256( + _mm256_shuffle_ps(_mm256_castsi256_ps(a0), _mm256_castsi256_ps(a1), + 0x88)); + __m256i o = _mm256_castps_si256( + _mm256_shuffle_ps(_mm256_castsi256_ps(a0), _mm256_castsi256_ps(a1), + 0xdd)); + __m256i d = _mm256_permute4x64_epi64(_mm256_avg_epu8(e, o), 0xd8); + _mm256_storeu_si256((__m256i*)dst_argb, d); + dst_argb += 32; + dst_width -= 8; + } +} +#endif // HAS_SCALEARGBROWDOWNEVEN_AVX2 + +#ifdef HAS_SCALEARGBCOLS_AVX2 +static const int32_t kStep0_7_Cols[8] = {0, 1, 2, 3, 4, 5, 6, 7}; + +LIBYUV_TARGET_AVX2 +void ScaleARGBCols_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + int n = dst_width & ~7; + int r = dst_width & 7; + if (n > 0) { + SIMD_ALIGNED(int32_t xi_buf[8]); + SIMD_ALIGNED(uint32_t pix_buf[8]); + __m256i vx = _mm256_add_epi32( + _mm256_set1_epi32(x), + _mm256_mullo_epi32( + _mm256_set1_epi32(dx), + _mm256_loadu_si256((const __m256i*)kStep0_7_Cols))); + const __m256i vdx8 = _mm256_slli_epi32(_mm256_set1_epi32(dx), 3); + for (int i = 0; i < n; i += 8) { + _mm256_store_si256((__m256i*)xi_buf, _mm256_srai_epi32(vx, 16)); + vx = _mm256_add_epi32(vx, vdx8); + pix_buf[0] = LoadU32(src_argb + (ptrdiff_t)xi_buf[0] * 4); + pix_buf[1] = LoadU32(src_argb + (ptrdiff_t)xi_buf[1] * 4); + pix_buf[2] = LoadU32(src_argb + (ptrdiff_t)xi_buf[2] * 4); + pix_buf[3] = LoadU32(src_argb + (ptrdiff_t)xi_buf[3] * 4); + pix_buf[4] = LoadU32(src_argb + (ptrdiff_t)xi_buf[4] * 4); + pix_buf[5] = LoadU32(src_argb + (ptrdiff_t)xi_buf[5] * 4); + pix_buf[6] = LoadU32(src_argb + (ptrdiff_t)xi_buf[6] * 4); + pix_buf[7] = LoadU32(src_argb + (ptrdiff_t)xi_buf[7] * 4); + _mm256_storeu_si256((__m256i*)dst_argb, + _mm256_load_si256((const __m256i*)pix_buf)); + dst_argb += 32; + } + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + const uint8_t* src = src_argb + (ptrdiff_t)(x >> 16) * 4; + dst_argb[0] = src[0]; + dst_argb[1] = src[1]; + dst_argb[2] = src[2]; + dst_argb[3] = src[3]; + dst_argb += 4; + x = (int)((uint32_t)x + (uint32_t)dx); + } +} +#endif // HAS_SCALEARGBCOLS_AVX2 + +#ifdef HAS_SCALEARGBCOLSUP2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleARGBColsUp2_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + (void)x; + (void)dx; + while (dst_width > 0) { + __m256i s = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_argb)), + 0xd8); + src_argb += 16; + _mm256_storeu_si256((__m256i*)dst_argb, _mm256_unpacklo_epi32(s, s)); + dst_argb += 32; + dst_width -= 8; + } +} +#endif // HAS_SCALEARGBCOLSUP2_AVX2 + +#ifdef HAS_SCALEARGBFILTERCOLS_AVX2 +static const int32_t kStep0_3_ARGB[4] = {0, 1, 2, 3}; +static const uvec8 kShuffleColARGB = { + 0u, 4u, 1u, 5u, 2u, 6u, 3u, 7u, + 8u, 12u, 9u, 13u, 10u, 14u, 11u, 15u}; +static const uint8_t kShuffleFractionsAVX2[32] = { + 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, 4u, + 8u, 8u, 8u, 8u, 8u, 8u, 8u, 8u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, +}; + +LIBYUV_TARGET_AVX2 +void ScaleARGBFilterCols_AVX2(uint8_t* dst_argb, + const uint8_t* src_argb, + int dst_width, + int x, + int dx) { + int n = dst_width & ~3; + int r = dst_width & 3; + if (n > 0) { + SIMD_ALIGNED(int32_t xi_buf[4]); + SIMD_ALIGNED(uint64_t pix_buf[4]); + const __m256i shuf_argb = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kShuffleColARGB)); + const __m256i shuf_frac = + _mm256_loadu_si256((const __m256i*)kShuffleFractionsAVX2); + const __m256i mask_7f = _mm256_set1_epi16(0x007f); + __m128i vx = _mm_add_epi32( + _mm_set1_epi32(x), + _mm_mullo_epi32(_mm_set1_epi32(dx), + _mm_loadu_si128((const __m128i*)kStep0_3_ARGB))); + const __m128i vdx4 = _mm_slli_epi32(_mm_set1_epi32(dx), 2); + for (int i = 0; i < n; i += 4) { + _mm_store_si128((__m128i*)xi_buf, _mm_srai_epi32(vx, 16)); + __m128i f128 = _mm_srli_epi16(vx, 9); + vx = _mm_add_epi32(vx, vdx4); + pix_buf[0] = LoadU64(src_argb + (ptrdiff_t)xi_buf[0] * 4); + pix_buf[1] = LoadU64(src_argb + (ptrdiff_t)xi_buf[1] * 4); + pix_buf[2] = LoadU64(src_argb + (ptrdiff_t)xi_buf[2] * 4); + pix_buf[3] = LoadU64(src_argb + (ptrdiff_t)xi_buf[3] * 4); + __m256i pix = _mm256_shuffle_epi8( + _mm256_load_si256((const __m256i*)pix_buf), shuf_argb); + __m256i weights = _mm256_xor_si256( + _mm256_shuffle_epi8( + _mm256_inserti128_si256(_mm256_zextsi128_si256(f128), f128, 1), + shuf_frac), + mask_7f); + __m256i res = _mm256_srli_epi16(_mm256_maddubs_epi16(pix, weights), 7); + res = _mm256_permute4x64_epi64(_mm256_packus_epi16(res, res), 0xd8); + _mm_storeu_si128((__m128i*)dst_argb, _mm256_castsi256_si128(res)); + dst_argb += 16; + } + x = (int)((uint32_t)x + (uint32_t)n * (uint32_t)dx); + } + for (int j = 0; j < r; ++j) { + ptrdiff_t xi = (ptrdiff_t)(x >> 16) * 4; + int xf = (x >> 9) & 0x7f; + const uint8_t* a = src_argb + xi; + const uint8_t* b = a + 4; + dst_argb[0] = (uint8_t)((a[0] * (0x7f ^ xf) + b[0] * xf) >> 7); + dst_argb[1] = (uint8_t)((a[1] * (0x7f ^ xf) + b[1] * xf) >> 7); + dst_argb[2] = (uint8_t)((a[2] * (0x7f ^ xf) + b[2] * xf) >> 7); + dst_argb[3] = (uint8_t)((a[3] * (0x7f ^ xf) + b[3] * xf) >> 7); + dst_argb += 4; + x = (int)((uint32_t)x + (uint32_t)dx); + } +} +#endif // HAS_SCALEARGBFILTERCOLS_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowDown2_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + while (dst_width >= 16) { + __m256i s0 = + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_uv), 16); + __m256i s1 = _mm256_srli_epi32( + _mm256_loadu_si256((const __m256i*)(src_uv + 32)), 16); + src_uv += 64; + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, d); + dst_uv += 32; + dst_width -= 16; + } + if (dst_width > 0) { + __m256i s0 = + _mm256_srli_epi32(_mm256_loadu_si256((const __m256i*)src_uv), 16); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s0), 0xd8); + _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(d)); + } +} +#endif // HAS_SCALEUVROWDOWN2_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2LINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowDown2Linear_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + const __m256i mask = _mm256_set1_epi32(0x0000ffff); + while (dst_width >= 16) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i r1 = _mm256_loadu_si256((const __m256i*)(src_uv + 32)); + src_uv += 64; + __m256i s0 = + _mm256_and_si256(_mm256_avg_epu8(r0, _mm256_srli_epi32(r0, 16)), mask); + __m256i s1 = + _mm256_and_si256(_mm256_avg_epu8(r1, _mm256_srli_epi32(r1, 16)), mask); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s1), 0xd8); + _mm256_storeu_si256((__m256i*)dst_uv, d); + dst_uv += 32; + dst_width -= 16; + } + if (dst_width > 0) { + __m256i r0 = _mm256_loadu_si256((const __m256i*)src_uv); + __m256i s0 = + _mm256_and_si256(_mm256_avg_epu8(r0, _mm256_srli_epi32(r0, 16)), mask); + __m256i d = _mm256_permute4x64_epi64(_mm256_packus_epi32(s0, s0), 0xd8); + _mm_storeu_si128((__m128i*)dst_uv, _mm256_castsi256_si128(d)); + } +} +#endif // HAS_SCALEUVROWDOWN2LINEAR_AVX2 + +#ifdef HAS_SCALEUVROWDOWN2BOX_AVX2 +static const uvec8 kShuffleSplitUV = {0u, 2u, 4u, 6u, 8u, 10u, 12u, 14u, + 1u, 3u, 5u, 7u, 9u, 11u, 13u, 15u}; +static const uvec8 kShuffleMergeUV = {0u, 8u, 2u, 10u, 4u, 12u, + 6u, 14u, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80}; + +LIBYUV_TARGET_AVX2 +void ScaleUVRowDown2Box_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + int dst_width) { + const __m256i split = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kShuffleSplitUV)); + const __m256i merge = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kShuffleMergeUV)); + const __m256i ones = _mm256_set1_epi8(1); + const __m256i zero = _mm256_setzero_si256(); + while (dst_width > 0) { + __m256i r0 = _mm256_maddubs_epi16( + _mm256_shuffle_epi8(_mm256_loadu_si256((const __m256i*)src_ptr), split), + ones); + __m256i r1 = _mm256_maddubs_epi16( + _mm256_shuffle_epi8( + _mm256_loadu_si256((const __m256i*)(src_ptr + src_stride)), split), + ones); + src_ptr += 32; + __m256i s = + _mm256_avg_epu16(_mm256_srli_epi16(_mm256_add_epi16(r0, r1), 1), zero); + __m256i d = _mm256_permute4x64_epi64(_mm256_shuffle_epi8(s, merge), 0xd8); + _mm_storeu_si128((__m128i*)dst_ptr, _mm256_castsi256_si128(d)); + dst_ptr += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVROWDOWN2BOX_AVX2 + +#ifdef HAS_SCALEUVROWDOWNEVEN_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowDownEven_AVX2(const uint8_t* src_uv, + ptrdiff_t src_stride, + int src_stepx, + uint8_t* dst_uv, + int dst_width) { + (void)src_stride; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 2; + SIMD_ALIGNED(uint16_t uv_buf[8]); + while (dst_width > 0) { + uv_buf[0] = LoadU16(src_uv); + uv_buf[1] = LoadU16(src_uv + step); + uv_buf[2] = LoadU16(src_uv + step * 2); + uv_buf[3] = LoadU16(src_uv + step * 3); + uv_buf[4] = LoadU16(src_uv + step * 4); + uv_buf[5] = LoadU16(src_uv + step * 5); + uv_buf[6] = LoadU16(src_uv + step * 6); + uv_buf[7] = LoadU16(src_uv + step * 7); + src_uv += step * 8; + _mm_storeu_si128((__m128i*)dst_uv, _mm_load_si128((const __m128i*)uv_buf)); + dst_uv += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVROWDOWNEVEN_AVX2 + +#if defined(HAS_SCALEUVROWUP2_LINEAR_AVX2) || \ + defined(HAS_SCALEUVROWUP2_BILINEAR_AVX2) +static const uvec8 kUVLinearMadd31 = {3, 1, 3, 1, 1, 3, 1, 3, + 3, 1, 3, 1, 1, 3, 1, 3}; +#endif + +#ifdef HAS_SCALEUVROWUP2_LINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowUp2_Linear_AVX2(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int dst_width) { + const __m256i madd31 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kUVLinearMadd31)); + const __m256i round2 = _mm256_set1_epi16(2); + while (dst_width > 0) { + __m256i s0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i s1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 2))), + 0xd8); + src_ptr += 16; + __m256i p = _mm256_unpacklo_epi8(s0, s1); + __m256i lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(p, p), madd31); + __m256i hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(p, p), madd31); + lo = _mm256_srli_epi16(_mm256_add_epi16(lo, round2), 2); + hi = _mm256_srli_epi16(_mm256_add_epi16(hi, round2), 2); + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(lo, hi)); + dst_ptr += 32; + dst_width -= 16; + } +} +#endif // HAS_SCALEUVROWUP2_LINEAR_AVX2 + +#ifdef HAS_SCALEUVROWUP2_BILINEAR_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowUp2_Bilinear_AVX2(const uint8_t* src_ptr, + ptrdiff_t src_stride, + uint8_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i madd31 = _mm256_broadcastsi128_si256( + _mm_loadu_si128((const __m128i*)&kUVLinearMadd31)); + const __m256i round8 = _mm256_set1_epi16(8); + while (dst_width > 0) { + __m256i a0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)src_ptr)), 0xd8); + __m256i a1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256(_mm_loadu_si128((const __m128i*)(src_ptr + 2))), + 0xd8); + __m256i pa = _mm256_unpacklo_epi8(a0, a1); + __m256i r0_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(pa, pa), madd31); + __m256i r0_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(pa, pa), madd31); + + __m256i b0 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))), + 0xd8); + __m256i b1 = _mm256_permute4x64_epi64( + _mm256_zextsi128_si256( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 2))), + 0xd8); + src_ptr += 16; + __m256i pb = _mm256_unpacklo_epi8(b0, b1); + __m256i r1_lo = _mm256_maddubs_epi16(_mm256_unpacklo_epi32(pb, pb), madd31); + __m256i r1_hi = _mm256_maddubs_epi16(_mm256_unpackhi_epi32(pb, pb), madd31); + + __m256i d0_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_lo, r0_lo), r0_lo), + _mm256_add_epi16(r1_lo, round8)), + 4); + __m256i d1_lo = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_lo, r1_lo), r1_lo), + _mm256_add_epi16(r0_lo, round8)), + 4); + __m256i d0_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r0_hi, r0_hi), r0_hi), + _mm256_add_epi16(r1_hi, round8)), + 4); + __m256i d1_hi = _mm256_srli_epi16( + _mm256_add_epi16(_mm256_add_epi16(_mm256_add_epi16(r1_hi, r1_hi), r1_hi), + _mm256_add_epi16(r0_hi, round8)), + 4); + + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi16(d0_lo, d0_hi)); + _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), + _mm256_packus_epi16(d1_lo, d1_hi)); + dst_ptr += 32; + dst_width -= 16; + } +} +#endif // HAS_SCALEUVROWUP2_BILINEAR_AVX2 + +#ifdef HAS_SCALEUVROWUP2_LINEAR_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowUp2_Linear_16_AVX2(const uint16_t* src_ptr, + uint16_t* dst_ptr, + int dst_width) { + const __m256i round2 = _mm256_set1_epi32(2); + while (dst_width > 0) { + __m256i near_lo = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i near_hi = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 2))); + src_ptr += 8; + __m256i far_lo = _mm256_shuffle_epi32(near_lo, 0x4e); + __m256i far_hi = _mm256_shuffle_epi32(near_hi, 0x4e); + __m256i d_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(near_lo, near_lo), + _mm256_add_epi32(near_lo, _mm256_add_epi32(far_lo, round2))), + 2); + __m256i d_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(near_hi, near_hi), + _mm256_add_epi32(near_hi, _mm256_add_epi32(far_hi, round2))), + 2); + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi32(d_lo, d_hi)); + dst_ptr += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVROWUP2_LINEAR_16_AVX2 + +#ifdef HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 +LIBYUV_TARGET_AVX2 +void ScaleUVRowUp2_Bilinear_16_AVX2(const uint16_t* src_ptr, + ptrdiff_t src_stride, + uint16_t* dst_ptr, + ptrdiff_t dst_stride, + int dst_width) { + const __m256i round8 = _mm256_set1_epi32(8); + while (dst_width > 0) { + __m256i a0 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)src_ptr)); + __m256i a1 = + _mm256_cvtepu16_epi32(_mm_loadu_si128((const __m128i*)(src_ptr + 2))); + __m256i r0_lo = _mm256_add_epi32( + _mm256_add_epi32(a0, a0), + _mm256_add_epi32(a0, _mm256_shuffle_epi32(a0, 0x4e))); + __m256i r0_hi = _mm256_add_epi32( + _mm256_add_epi32(a1, a1), + _mm256_add_epi32(a1, _mm256_shuffle_epi32(a1, 0x4e))); + + __m256i b0 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride))); + __m256i b1 = _mm256_cvtepu16_epi32( + _mm_loadu_si128((const __m128i*)(src_ptr + src_stride + 2))); + src_ptr += 8; + __m256i r1_lo = _mm256_add_epi32( + _mm256_add_epi32(b0, b0), + _mm256_add_epi32(b0, _mm256_shuffle_epi32(b0, 0x4e))); + __m256i r1_hi = _mm256_add_epi32( + _mm256_add_epi32(b1, b1), + _mm256_add_epi32(b1, _mm256_shuffle_epi32(b1, 0x4e))); + + __m256i d0_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_lo, r0_lo), r0_lo), + _mm256_add_epi32(r1_lo, round8)), + 4); + __m256i d1_lo = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_lo, r1_lo), r1_lo), + _mm256_add_epi32(r0_lo, round8)), + 4); + __m256i d0_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r0_hi, r0_hi), r0_hi), + _mm256_add_epi32(r1_hi, round8)), + 4); + __m256i d1_hi = _mm256_srli_epi32( + _mm256_add_epi32(_mm256_add_epi32(_mm256_add_epi32(r1_hi, r1_hi), r1_hi), + _mm256_add_epi32(r0_hi, round8)), + 4); + + _mm256_storeu_si256((__m256i*)dst_ptr, _mm256_packus_epi32(d0_lo, d0_hi)); + _mm256_storeu_si256((__m256i*)(dst_ptr + dst_stride), + _mm256_packus_epi32(d1_lo, d1_hi)); + dst_ptr += 16; + dst_width -= 8; + } +} +#endif // HAS_SCALEUVROWUP2_BILINEAR_16_AVX2 + #undef LIBYUV_TARGET_AVX2 #undef LIBYUV_TARGET_AVX512BW #undef LIBYUV_TARGET_AVX512VBMI diff --git a/unit_test/compare_test.cc b/unit_test/compare_test.cc index 2556bedaa..ba0bd272c 100644 --- a/unit_test/compare_test.cc +++ b/unit_test/compare_test.cc @@ -208,6 +208,61 @@ TEST_F(LibYUVCompareTest, BenchmarkARGBDetect_Unaligned) { free_aligned_buffer_page_end(src_a); } +TEST_F(LibYUVCompareTest, TestARGBDetectAnyWidth) { + const int kMaxWidth = 65; + const int kBufSize = kMaxWidth * 4 + 4; + align_buffer_page_end(src_a, kBufSize); + memset(src_a, 255, kBufSize); + + for (int width = 1; width <= kMaxWidth; ++width) { + for (int offset = 0; offset < 4; ++offset) { + uint8_t* row = src_a + kBufSize - width * 4 - offset; + // All 255 -> 0 (unknown). + MaskCpuFlags(disable_cpu_flags_); + uint32_t c_fourcc = ARGBDetect(row, width * 4, width, 1); + MaskCpuFlags(benchmark_cpu_info_); + uint32_t opt_fourcc = ARGBDetect(row, width * 4, width, 1); + EXPECT_EQ(0u, c_fourcc); + EXPECT_EQ(c_fourcc, opt_fourcc) << "width " << width << " off " << offset; + + for (int x = 0; x < width; ++x) { + // Byte 0 != 255 -> FOURCC_BGRA. + row[x * 4 + 0] = 128; + MaskCpuFlags(disable_cpu_flags_); + c_fourcc = ARGBDetect(row, width * 4, width, 1); + MaskCpuFlags(benchmark_cpu_info_); + opt_fourcc = ARGBDetect(row, width * 4, width, 1); + EXPECT_EQ(static_cast(FOURCC_BGRA), c_fourcc); + EXPECT_EQ(c_fourcc, opt_fourcc) + << "width " << width << " x " << x << " byte 0"; + + // Both byte 0 and byte 3 != 255 -> FOURCC_BGRA (byte 0 checked first). + row[x * 4 + 3] = 64; + MaskCpuFlags(disable_cpu_flags_); + c_fourcc = ARGBDetect(row, width * 4, width, 1); + MaskCpuFlags(benchmark_cpu_info_); + opt_fourcc = ARGBDetect(row, width * 4, width, 1); + EXPECT_EQ(static_cast(FOURCC_BGRA), c_fourcc); + EXPECT_EQ(c_fourcc, opt_fourcc) + << "width " << width << " x " << x << " byte 0+3"; + + // Byte 3 != 255 only -> FOURCC_ARGB. + row[x * 4 + 0] = 255; + MaskCpuFlags(disable_cpu_flags_); + c_fourcc = ARGBDetect(row, width * 4, width, 1); + MaskCpuFlags(benchmark_cpu_info_); + opt_fourcc = ARGBDetect(row, width * 4, width, 1); + EXPECT_EQ(static_cast(FOURCC_ARGB), c_fourcc); + EXPECT_EQ(c_fourcc, opt_fourcc) + << "width " << width << " x " << x << " byte 3"; + row[x * 4 + 3] = 255; + } + } + } + + free_aligned_buffer_page_end(src_a); +} + #ifdef ENABLE_ROW_TESTS TEST_F(LibYUVCompareTest, BenchmarkHammingDistance_Opt) { const int kMaxWidth = 4096 * 3; diff --git a/unit_test/cpu_test.cc b/unit_test/cpu_test.cc index 1a08f609b..3829f8dd2 100644 --- a/unit_test/cpu_test.cc +++ b/unit_test/cpu_test.cc @@ -207,6 +207,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) { int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8); int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8); int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM); + int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32); printf("Has X86 0x%x\n", has_x86); printf("Has SSE2 0x%x\n", has_sse2); printf("Has SSSE3 0x%x\n", has_ssse3); @@ -230,6 +231,7 @@ TEST_F(LibYUVBaseTest, TestCpuHas) { printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8); printf("Has AMXINT8 0x%x\n", has_amxint8); printf("Has AVX512BMM 0x%x\n", has_avx512bmm); + printf("Has AMXTF32 0x%x\n", has_amxtf32); } #endif // defined(__i386__) || defined(__x86_64__) || defined(_M_IX86) || // defined(_M_X64) diff --git a/unit_test/planar_test.cc b/unit_test/planar_test.cc index 01312ff60..5f7cf116e 100644 --- a/unit_test/planar_test.cc +++ b/unit_test/planar_test.cc @@ -23,6 +23,11 @@ #include "libyuv/rotate.h" #include "libyuv/scale.h" +#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \ + !defined(ENABLE_ROW_TESTS) +#define ENABLE_ROW_TESTS +#endif + #ifdef ENABLE_ROW_TESTS // row.h defines SIMD_ALIGNED, overriding unit_test.h // TODO(fbarchard): Remove row.h from unittests. Test public functions. @@ -1573,21 +1578,21 @@ TEST_F(LibYUVPlanarTest, TestAffine) { ASSERT_EQ(96u, interpolate_pixels_C[128][0]); ASSERT_EQ(191u, interpolate_pixels_C[255][3]); -#if defined(HAS_ARGBAFFINEROW_SSE2) - int has_sse2 = TestCpuFlag(kCpuHasSSE2); - if (has_sse2) { +#if defined(HAS_ARGBAFFINEROW_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { SIMD_ALIGNED(uint8_t interpolate_pixels_Opt[1280][4]); - ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride, + ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride, &interpolate_pixels_Opt[0][0], uv_step, 1280); ASSERT_EQ(0, memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1280 * 4)); - ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride, + ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride, &interpolate_pixels_Opt[0][0], uv_step, 1279); ASSERT_EQ(0, memcmp(interpolate_pixels_Opt, interpolate_pixels_C, 1279 * 4)); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { - ARGBAffineRow_SSE2(&orig_pixels_0[0][0][0], stride, + ARGBAffineRow_AVX2(&orig_pixels_0[0][0][0], stride, &interpolate_pixels_Opt[0][0], uv_step, 1280); } } @@ -3998,10 +4003,10 @@ TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 12) TESTTPLANARTOP(MergeXR30, uint16_t, uint8_t, 16) // TODO(fbarchard): improve test for platforms and cpu detect -#ifdef HAS_MERGEUVROW_16_AVX2 +#if defined(HAS_MERGEUVROW_16_AVX512BW) || defined(HAS_MERGEUVROW_16_AVX2) TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) { - // Round count up to multiple of 8 - const int kPixels = (benchmark_width_ * benchmark_height_ + 7) & ~7; + // Round count up to multiple of 16 + const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15; align_buffer_page_end(src_pixels_u, kPixels * 2); align_buffer_page_end(src_pixels_v, kPixels * 2); @@ -4017,14 +4022,30 @@ TEST_F(LibYUVPlanarTest, MergeUVRow_16_Opt) { reinterpret_cast(src_pixels_v), reinterpret_cast(dst_pixels_uv_c), 16, kPixels); +#if defined(HAS_MERGEUVROW_16_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_MERGEUVROW_16_AVX2) int has_avx2 = TestCpuFlag(kCpuHasAVX2); +#endif for (int i = 0; i < benchmark_iterations_; ++i) { - if (has_avx2) { +#if defined(HAS_MERGEUVROW_16_AVX512BW) + if (has_avx512) { + MergeUVRow_16_AVX512BW(reinterpret_cast(src_pixels_u), + reinterpret_cast(src_pixels_v), + reinterpret_cast(dst_pixels_uv_opt), 16, + kPixels); + } else +#endif +#if defined(HAS_MERGEUVROW_16_AVX2) + if (has_avx2) { MergeUVRow_16_AVX2(reinterpret_cast(src_pixels_u), reinterpret_cast(src_pixels_v), reinterpret_cast(dst_pixels_uv_opt), 16, kPixels); - } else { + } else +#endif + { MergeUVRow_16_C(reinterpret_cast(src_pixels_u), reinterpret_cast(src_pixels_v), reinterpret_cast(dst_pixels_uv_opt), 16, @@ -4331,10 +4352,10 @@ TEST_F(LibYUVPlanarTest, UYVYToY) { #ifdef ENABLE_ROW_TESTS // TODO(fbarchard): Improve test for more platforms. -#ifdef HAS_CONVERT16TO8ROW_AVX2 +#if defined(HAS_CONVERT16TO8ROW_AVX512BW) || defined(HAS_CONVERT16TO8ROW_AVX2) TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) { - // AVX2 does multiple of 32, so round count up - const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31; + // AVX512 does multiple of 64, so round count up + const int kPixels = (benchmark_width_ * benchmark_height_ + 63) & ~63; align_buffer_page_end(src_pixels_y, kPixels * 2); align_buffer_page_end(dst_pixels_y_opt, kPixels); align_buffer_page_end(dst_pixels_y_c, kPixels); @@ -4351,16 +4372,26 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) { Convert16To8Row_C(reinterpret_cast(src_pixels_y), dst_pixels_y_c, 16384, kPixels); +#if defined(HAS_CONVERT16TO8ROW_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_CONVERT16TO8ROW_AVX2) int has_avx2 = TestCpuFlag(kCpuHasAVX2); - int has_ssse3 = TestCpuFlag(kCpuHasSSSE3); +#endif for (int i = 0; i < benchmark_iterations_; ++i) { - if (has_avx2) { +#if defined(HAS_CONVERT16TO8ROW_AVX512BW) + if (has_avx512) { + Convert16To8Row_AVX512BW(reinterpret_cast(src_pixels_y), + dst_pixels_y_opt, 16384, kPixels); + } else +#endif +#if defined(HAS_CONVERT16TO8ROW_AVX2) + if (has_avx2) { Convert16To8Row_AVX2(reinterpret_cast(src_pixels_y), dst_pixels_y_opt, 16384, kPixels); - } else if (has_ssse3) { - Convert16To8Row_SSSE3(reinterpret_cast(src_pixels_y), - dst_pixels_y_opt, 16384, kPixels); - } else { + } else +#endif + { Convert16To8Row_C(reinterpret_cast(src_pixels_y), dst_pixels_y_opt, 16384, kPixels); } @@ -4374,7 +4405,7 @@ TEST_F(LibYUVPlanarTest, Convert16To8Row_Opt) { free_aligned_buffer_page_end(dst_pixels_y_opt); free_aligned_buffer_page_end(dst_pixels_y_c); } -#endif // HAS_CONVERT16TO8ROW_AVX2 +#endif // HAS_CONVERT16TO8ROW_AVX512BW || HAS_CONVERT16TO8ROW_AVX2 #if defined(HAS_HALFWIDTHROW_16TO8_AVX2) || \ defined(HAS_HALFWIDTHROW_16TO8_SSSE3) || \ @@ -4481,8 +4512,8 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) { #if defined(HAS_HALFWIDTHROW_16TO8_AVX512BW) if (has_avx512) { - HalfWidthRow_16To8_Any_AVX512BW(src_pixels_uv, kMaxPixels * 2, - dst_pixels_uv_opt, 16384, width); + HalfWidthRow_16To8_AVX512BW(src_pixels_uv, kMaxPixels * 2, + dst_pixels_uv_opt, 16384, width); } else #endif #if defined(HAS_HALFWIDTHROW_16TO8_AVX2) @@ -4519,10 +4550,11 @@ TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Any) { } #endif -#ifdef HAS_UYVYTOYROW_NEON +#if defined(HAS_UYVYTOYROW_AVX512BW) || defined(HAS_UYVYTOYROW_AVX2) || \ + defined(HAS_UYVYTOYROW_NEON) TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) { - // NEON does multiple of 16, so round count up - const int kPixels = (benchmark_width_ * benchmark_height_ + 15) & ~15; + // AVX2/AVX512 does multiple of 32, so round count up + const int kPixels = (benchmark_width_ * benchmark_height_ + 31) & ~31; align_buffer_page_end(src_pixels_y, kPixels * 2); align_buffer_page_end(dst_pixels_y_opt, kPixels); align_buffer_page_end(dst_pixels_y_c, kPixels); @@ -4533,8 +4565,34 @@ TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) { UYVYToYRow_C(src_pixels_y, dst_pixels_y_c, kPixels); +#if defined(HAS_UYVYTOYROW_AVX512BW) + int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); +#endif +#if defined(HAS_UYVYTOYROW_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); +#endif +#if defined(HAS_UYVYTOYROW_NEON) + int has_neon = TestCpuFlag(kCpuHasNEON); +#endif for (int i = 0; i < benchmark_iterations_; ++i) { - UYVYToYRow_NEON(src_pixels_y, dst_pixels_y_opt, kPixels); +#if defined(HAS_UYVYTOYROW_AVX512BW) + if (has_avx512) { + UYVYToYRow_AVX512BW(src_pixels_y, dst_pixels_y_opt, kPixels); + } else +#endif +#if defined(HAS_UYVYTOYROW_AVX2) + if (has_avx2) { + UYVYToYRow_AVX2(src_pixels_y, dst_pixels_y_opt, kPixels); + } else +#endif +#if defined(HAS_UYVYTOYROW_NEON) + if (has_neon) { + UYVYToYRow_NEON(src_pixels_y, dst_pixels_y_opt, kPixels); + } else +#endif + { + UYVYToYRow_C(src_pixels_y, dst_pixels_y_opt, kPixels); + } } for (int i = 0; i < kPixels; ++i) { @@ -4545,7 +4603,99 @@ TEST_F(LibYUVPlanarTest, UYVYToYRow_Opt) { free_aligned_buffer_page_end(dst_pixels_y_opt); free_aligned_buffer_page_end(dst_pixels_y_c); } -#endif // HAS_UYVYTOYROW_NEON +#endif // HAS_UYVYTOYROW_... + +#if defined(HAS_RGB565TOYROW_AVX2) && defined(HAS_ARGB1555TOYROW_AVX2) && \ + defined(HAS_ARGB4444TOYROW_AVX2) +TEST_F(LibYUVPlanarTest, RGB16ToYUVRow_Opt) { + if (!TestCpuFlag(kCpuHasAVX2)) { + return; + } + const int kMaxWidth = 128; + SIMD_ALIGNED(uint8_t src0[kMaxWidth * 2]); + SIMD_ALIGNED(uint8_t src1[kMaxWidth * 2]); + SIMD_ALIGNED(uint8_t dst_y_c[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_y_opt[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_u_c[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_v_c[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_u_opt[kMaxWidth]); + SIMD_ALIGNED(uint8_t dst_v_opt[kMaxWidth]); + MemRandomize(src0, sizeof(src0)); + MemRandomize(src1, sizeof(src1)); + + for (int width : {1, 2, 7, 15, 16, 17, 31, 32, 33, 64, 127, 128}) { + const int halfwidth = (width + 1) / 2; + const int stride = kMaxWidth * 2; + + RGB565ToYRow_C(src0, dst_y_c, width); + if (IS_ALIGNED(width, 32)) { + RGB565ToYRow_AVX2(src0, dst_y_opt, width); + } else { + RGB565ToYRow_Any_AVX2(src0, dst_y_opt, width); + } + for (int i = 0; i < width; ++i) { + ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1); + } + + ARGB1555ToYRow_C(src0, dst_y_c, width); + if (IS_ALIGNED(width, 32)) { + ARGB1555ToYRow_AVX2(src0, dst_y_opt, width); + } else { + ARGB1555ToYRow_Any_AVX2(src0, dst_y_opt, width); + } + for (int i = 0; i < width; ++i) { + ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1); + } + + ARGB4444ToYRow_C(src0, dst_y_c, width); + if (IS_ALIGNED(width, 32)) { + ARGB4444ToYRow_AVX2(src0, dst_y_opt, width); + } else { + ARGB4444ToYRow_Any_AVX2(src0, dst_y_opt, width); + } + for (int i = 0; i < width; ++i) { + ASSERT_NEAR(dst_y_c[i], dst_y_opt[i], 1); + } + + SIMD_ALIGNED(uint8_t src2rows[kMaxWidth * 2 * 2]); + memcpy(src2rows, src0, stride); + memcpy(src2rows + stride, src1, stride); + + RGB565ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width); + if (IS_ALIGNED(width, 32)) { + RGB565ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } else { + RGB565ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } + for (int i = 0; i < halfwidth; ++i) { + ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2); + ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2); + } + + ARGB1555ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width); + if (IS_ALIGNED(width, 32)) { + ARGB1555ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } else { + ARGB1555ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } + for (int i = 0; i < halfwidth; ++i) { + ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2); + ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2); + } + + ARGB4444ToUVRow_C(src2rows, stride, dst_u_c, dst_v_c, width); + if (IS_ALIGNED(width, 32)) { + ARGB4444ToUVRow_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } else { + ARGB4444ToUVRow_Any_AVX2(src2rows, stride, dst_u_opt, dst_v_opt, width); + } + for (int i = 0; i < halfwidth; ++i) { + ASSERT_NEAR(dst_u_c[i], dst_u_opt[i], 2); + ASSERT_NEAR(dst_v_c[i], dst_v_opt[i], 2); + } + } +} +#endif #endif // ENABLE_ROW_TESTS @@ -4601,7 +4751,6 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) { int has_avx512 = TestCpuFlag(kCpuHasAVX512BW); #endif int has_avx2 = TestCpuFlag(kCpuHasAVX2); - int has_sse2 = TestCpuFlag(kCpuHasSSE2); for (int i = 0; i < benchmark_iterations_; ++i) { #if defined(HAS_CONVERT8TO16ROW_AVX512BW) if (has_avx512) { @@ -4614,10 +4763,6 @@ TEST_F(LibYUVPlanarTest, Convert8To16Row_Opt) { Convert8To16Row_AVX2(src_pixels_y, reinterpret_cast(dst_pixels_y_opt), 10, kPixels); - } else if (has_sse2) { - Convert8To16Row_SSE2(src_pixels_y, - reinterpret_cast(dst_pixels_y_opt), 10, - kPixels); } else { Convert8To16Row_C(src_pixels_y, reinterpret_cast(dst_pixels_y_opt), 10, @@ -4666,6 +4811,16 @@ float TestScaleMaxSamples(int benchmark_width, max_opt = ScaleMaxSamples_NEON(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); +#elif defined(HAS_SCALEMAXSAMPLES_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + max_opt = ScaleMaxSamples_AVX2(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else { + max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #else max_opt = ScaleMaxSamples_C(reinterpret_cast(orig_y), @@ -4734,6 +4889,16 @@ float TestScaleSumSamples(int benchmark_width, sum_opt = ScaleSumSamples_NEON(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); +#elif defined(HAS_SCALESUMSAMPLES_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + sum_opt = ScaleSumSamples_AVX2(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } else { + sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, + kPixels); + } #else sum_opt = ScaleSumSamples_C(reinterpret_cast(orig_y), @@ -4811,6 +4976,14 @@ float TestScaleSamples(int benchmark_width, #ifdef HAS_SCALESUMSAMPLES_NEON ScaleSamples_NEON(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); +#elif defined(HAS_SCALESAMPLES_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + ScaleSamples_AVX2(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } else { + ScaleSamples_C(reinterpret_cast(orig_y), + reinterpret_cast(dst_opt), scale, kPixels); + } #else ScaleSamples_C(reinterpret_cast(orig_y), reinterpret_cast(dst_opt), scale, kPixels); @@ -4923,7 +5096,14 @@ TEST_F(LibYUVPlanarTest, TestGaussRow_Opt) { } GaussRow_C(&orig_pixels[0], &dst_pixels_c[0], 1280); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { -#if !defined(LIBYUV_DISABLE_NEON) && \ +#if defined(HAS_GAUSSROW_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + GaussRow_AVX2(&orig_pixels[0], &dst_pixels_opt[0], 1280); + } else { + GaussRow_C(&orig_pixels[0], &dst_pixels_opt[0], 1280); + } +#elif !defined(LIBYUV_DISABLE_NEON) && \ (defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON)) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { @@ -4977,7 +5157,18 @@ TEST_F(LibYUVPlanarTest, TestGaussCol_Opt) { &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], &dst_pixels_c[0], 1280); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { -#if !defined(LIBYUV_DISABLE_NEON) && \ +#if defined(HAS_GAUSSCOL_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + GaussCol_AVX2(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2], + &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], + &dst_pixels_opt[0], 1280); + } else { + GaussCol_C(&orig_pixels[0], &orig_pixels[1280], &orig_pixels[1280 * 2], + &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], + &dst_pixels_opt[0], 1280); + } +#elif !defined(LIBYUV_DISABLE_NEON) && \ (defined(__aarch64__) || defined(__ARM_NEON__) || defined(LIBYUV_NEON)) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { @@ -5015,7 +5206,14 @@ TEST_F(LibYUVPlanarTest, TestGaussRow_F32_Opt) { } GaussRow_F32_C(&orig_pixels[0], &dst_pixels_c[0], 1280); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { -#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) +#if defined(HAS_GAUSSROW_F32_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + GaussRow_F32_AVX2(&orig_pixels[0], &dst_pixels_opt[0], 1280); + } else { + GaussRow_F32_C(&orig_pixels[0], &dst_pixels_opt[0], 1280); + } +#elif !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { GaussRow_F32_NEON(&orig_pixels[0], &dst_pixels_opt[0], 1280); @@ -5049,7 +5247,18 @@ TEST_F(LibYUVPlanarTest, TestGaussCol_F32_Opt) { &orig_pixels[1280 * 3], &orig_pixels[1280 * 4], &dst_pixels_c[0], 1280); for (int i = 0; i < benchmark_pixels_div1280_; ++i) { -#if !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) +#if defined(HAS_GAUSSCOL_F32_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + GaussCol_F32_AVX2(&orig_pixels[0], &orig_pixels[1280], + &orig_pixels[1280 * 2], &orig_pixels[1280 * 3], + &orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280); + } else { + GaussCol_F32_C(&orig_pixels[0], &orig_pixels[1280], + &orig_pixels[1280 * 2], &orig_pixels[1280 * 3], + &orig_pixels[1280 * 4], &dst_pixels_opt[0], 1280); + } +#elif !defined(LIBYUV_DISABLE_NEON) && defined(__aarch64__) int has_neon = TestCpuFlag(kCpuHasNEON); if (has_neon) { GaussCol_F32_NEON(&orig_pixels[0], &orig_pixels[1280], @@ -5129,6 +5338,151 @@ TEST_F(LibYUVPlanarTest, SwapUVRow) { free_aligned_buffer_page_end(src_pixels_vu); free_aligned_buffer_page_end(dst_pixels_uv); } + +TEST_F(LibYUVPlanarTest, HalfRow_Opt) { + const int kMaxPixels = 160; + align_buffer_page_end(src_pixels, kMaxPixels * 2); + align_buffer_page_end(dst_pixels_opt, kMaxPixels); + align_buffer_page_end(dst_pixels_c, kMaxPixels); + MemRandomize(src_pixels, kMaxPixels * 2); + + for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels}}) { + for (int width = 1; width <= 129; ++width) { + memset(dst_pixels_opt, 0, kMaxPixels); + memset(dst_pixels_c, 1, kMaxPixels); + HalfRow_C(src_pixels, stride, dst_pixels_c, width); +#if defined(HAS_HALFROW_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + if (IS_ALIGNED(width, 32)) { + HalfRow_AVX2(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_Any_AVX2(src_pixels, stride, dst_pixels_opt, width); + } + } else +#endif + { + HalfRow_C(src_pixels, stride, dst_pixels_opt, width); + } + ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width)) + << "stride=" << stride << " width=" << width; + } + } + + free_aligned_buffer_page_end(src_pixels); + free_aligned_buffer_page_end(dst_pixels_opt); + free_aligned_buffer_page_end(dst_pixels_c); +} + +TEST_F(LibYUVPlanarTest, HalfRow_16_Opt) { + const int kMaxPixels = 160; + align_buffer_page_end_16(src_pixels, kMaxPixels * 2); + align_buffer_page_end_16(dst_pixels_opt, kMaxPixels); + align_buffer_page_end_16(dst_pixels_c, kMaxPixels); + for (int i = 0; i < kMaxPixels * 2; ++i) { + src_pixels[i] = static_cast(fastrand()); + } + + for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels}}) { + for (int width = 1; width <= 129; ++width) { + memset(dst_pixels_opt, 0, kMaxPixels * 2); + memset(dst_pixels_c, 1, kMaxPixels * 2); + HalfRow_16_C(src_pixels, stride, dst_pixels_c, width); +#if defined(HAS_HALFROW_16_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + if (IS_ALIGNED(width, 16)) { + HalfRow_16_AVX2(src_pixels, stride, dst_pixels_opt, width); + } else { + HalfRow_16_Any_AVX2(src_pixels, stride, dst_pixels_opt, width); + } + } else +#endif + { + HalfRow_16_C(src_pixels, stride, dst_pixels_opt, width); + } + ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width * 2)) + << "stride=" << stride << " width=" << width; + } + } + + free_aligned_buffer_page_end_16(src_pixels); + free_aligned_buffer_page_end_16(dst_pixels_opt); + free_aligned_buffer_page_end_16(dst_pixels_c); +} + +TEST_F(LibYUVPlanarTest, HalfWidthRow_16To8_Odd_Opt) { + const int kMaxPixels = 160; + align_buffer_page_end_16(src_pixels, kMaxPixels * 4); + align_buffer_page_end(dst_pixels_opt, kMaxPixels); + align_buffer_page_end(dst_pixels_c, kMaxPixels); + for (int i = 0; i < kMaxPixels * 4; ++i) { + src_pixels[i] = fastrand() & 1023; + } + + for (ptrdiff_t stride : {ptrdiff_t{0}, ptrdiff_t{kMaxPixels * 2}}) { + for (int width = 1; width <= 129; ++width) { + memset(dst_pixels_opt, 0, kMaxPixels); + memset(dst_pixels_c, 1, kMaxPixels); + HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_c, 16384, width); +#if defined(HAS_HALFWIDTHROW_16TO8_ODD_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + HalfWidthRow_16To8_Odd_AVX2(src_pixels, stride, dst_pixels_opt, 16384, + width); + } else +#endif + { + HalfWidthRow_16To8_Odd_C(src_pixels, stride, dst_pixels_opt, 16384, + width); + } + ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width)) + << "stride=" << stride << " width=" << width; + } + } + + free_aligned_buffer_page_end_16(src_pixels); + free_aligned_buffer_page_end(dst_pixels_opt); + free_aligned_buffer_page_end(dst_pixels_c); +} + +TEST_F(LibYUVPlanarTest, ARGBChannelReorderRow_C_Test) { + const int kPixels = 65; + align_buffer_page_end(src_argb, kPixels * 4); + align_buffer_page_end(dst_row, kPixels * 4); + align_buffer_page_end(dst_api, kPixels * 4); + MemRandomize(src_argb, kPixels * 4); + + ARGBToABGRRow_C(src_argb, dst_row, kPixels); + ARGBToABGR(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4)); + + ARGBToBGRARow_C(src_argb, dst_row, kPixels); + ARGBToBGRA(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4)); + + ARGBToRGBARow_C(src_argb, dst_row, kPixels); + ARGBToRGBA(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4)); + + RGBAToARGBRow_C(src_argb, dst_row, kPixels); + RGBAToARGB(src_argb, kPixels * 4, dst_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_row, dst_api, kPixels * 4)); + + align_buffer_page_end_16(src_ar64, kPixels * 4); + align_buffer_page_end_16(dst_ab64_row, kPixels * 4); + align_buffer_page_end_16(dst_ab64_api, kPixels * 4); + for (int i = 0; i < kPixels * 4; ++i) { + src_ar64[i] = static_cast(fastrand()); + } + AR64ToAB64Row_C(src_ar64, dst_ab64_row, kPixels); + AR64ToAB64(src_ar64, kPixels * 4, dst_ab64_api, kPixels * 4, kPixels, 1); + ASSERT_EQ(0, memcmp(dst_ab64_row, dst_ab64_api, kPixels * 8)); + + free_aligned_buffer_page_end_16(src_ar64); + free_aligned_buffer_page_end_16(dst_ab64_row); + free_aligned_buffer_page_end_16(dst_ab64_api); + free_aligned_buffer_page_end(src_argb); + free_aligned_buffer_page_end(dst_row); + free_aligned_buffer_page_end(dst_api); +} #endif // ENABLE_ROW_TESTS TEST_F(LibYUVPlanarTest, TestGaussPlane_F32) { diff --git a/unit_test/rotate_test.cc b/unit_test/rotate_test.cc index b9ece9409..5ddf899cb 100644 --- a/unit_test/rotate_test.cc +++ b/unit_test/rotate_test.cc @@ -14,6 +14,11 @@ #include "libyuv/cpu_id.h" #include "libyuv/rotate.h" +#if !defined(LIBYUV_USING_SHARED_LIBRARY) && !defined(COMPONENT_BUILD) && \ + !defined(ENABLE_ROW_TESTS) +#define ENABLE_ROW_TESTS +#endif + #ifdef ENABLE_ROW_TESTS #include "libyuv/rotate_row.h" #endif @@ -892,9 +897,9 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Test) { TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4, (uint8_t*)dst_pixels_opt, width * 4, width); } else -#elif defined(HAS_TRANSPOSEWX4_32_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - TransposeWx4_32_SSE2((const uint8_t*)src_pixels, height * 4, +#elif defined(HAS_TRANSPOSEWX4_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx4_32_AVX2((const uint8_t*)src_pixels, height * 4, (uint8_t*)dst_pixels_opt, width * 4, width); } else #endif @@ -933,9 +938,9 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Opt) { TransposeWx4_32_NEON((const uint8_t*)src_pixels, height * 4, (uint8_t*)dst_pixels_opt, width * 4, width); } else -#elif defined(HAS_TRANSPOSEWX4_32_SSE2) - if (TestCpuFlag(kCpuHasSSE2)) { - TransposeWx4_32_SSE2((const uint8_t*)src_pixels, height * 4, +#elif defined(HAS_TRANSPOSEWX4_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + TransposeWx4_32_AVX2((const uint8_t*)src_pixels, height * 4, (uint8_t*)dst_pixels_opt, width * 4, width); } else #endif @@ -954,6 +959,97 @@ TEST_F(LibYUVRotateTest, TransposeWx4_32_Opt) { free_aligned_buffer_page_end(dst_pixels_opt); } +TEST_F(LibYUVRotateTest, TransposeWx4_32_Any) { + for (int width : {4, 8, 12, 16, 20, 28, 36}) { + const int height = 4; + align_buffer_page_end(src_pixels, height * width * 4); + align_buffer_page_end(dst_pixels_c, width * height * 4); + align_buffer_page_end(dst_pixels_opt, width * height * 4); + + MemRandomize(src_pixels, height * width * 4); + memset(dst_pixels_c, 1, width * height * 4); + memset(dst_pixels_opt, 2, width * height * 4); + + TransposeWx4_32_C(src_pixels, height * 4, dst_pixels_c, width * 4, width); +#if defined(HAS_TRANSPOSEWX4_32_AVX2) + if (TestCpuFlag(kCpuHasAVX2)) { + if ((width & 7) == 0) { + TransposeWx4_32_AVX2(src_pixels, height * 4, dst_pixels_opt, width * 4, + width); + } else { + TransposeWx4_32_Any_AVX2(src_pixels, height * 4, dst_pixels_opt, + width * 4, width); + } + } else +#endif +#if defined(HAS_TRANSPOSEWX4_32_NEON) + if (TestCpuFlag(kCpuHasNEON)) { + TransposeWx4_32_NEON(src_pixels, height * 4, dst_pixels_opt, width * 4, + width); + } else +#endif + { + TransposeWx4_32_C(src_pixels, height * 4, dst_pixels_opt, width * 4, + width); + } + + ASSERT_EQ(0, memcmp(dst_pixels_c, dst_pixels_opt, width * height * 4)) + << "width=" << width; + + free_aligned_buffer_page_end(src_pixels); + free_aligned_buffer_page_end(dst_pixels_c); + free_aligned_buffer_page_end(dst_pixels_opt); + } +} + +#if defined(HAS_TRANSPOSEWXH_32_AVX2) +TEST_F(LibYUVRotateTest, TransposeWxH_32_Test) { + if (!TestCpuFlag(kCpuHasAVX2)) { + return; + } + const int kMaxW = 35; + const int kMaxH = 25; + align_buffer_page_end(src_pixels, kMaxW * kMaxH * 4); + align_buffer_page_end(dst_pixels_c, kMaxW * kMaxH * 4); + align_buffer_page_end(dst_pixels_opt, kMaxW * kMaxH * 4); + MemRandomize(src_pixels, kMaxW * kMaxH * 4); + + for (int h = 1; h <= kMaxH; ++h) { + for (int w = 1; w <= kMaxW; ++w) { + const int src_stride = w * 4; + const int dst_stride = h * 4; + const int src_bytes = src_stride * h; + const int dst_bytes = dst_stride * w; + const uint8_t* src = src_pixels + kMaxW * kMaxH * 4 - src_bytes; + uint8_t* dst_c = dst_pixels_c + kMaxW * kMaxH * 4 - dst_bytes; + uint8_t* dst_opt = dst_pixels_opt + kMaxW * kMaxH * 4 - dst_bytes; + + memset(dst_c, 1, dst_bytes); + memset(dst_opt, 2, dst_bytes); + TransposeWxH_32_C(src, src_stride, dst_c, dst_stride, w, h); + TransposeWxH_32_AVX2(src, src_stride, dst_opt, dst_stride, w, h); + ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << w << "x" << h; + + // Also test negative src_stride (90-degree) and negative dst_stride + // (270-degree). + memset(dst_c, 3, dst_bytes); + memset(dst_opt, 4, dst_bytes); + TransposeWxH_32_C(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride, + dst_c + (ptrdiff_t)(w - 1) * dst_stride, -dst_stride, w, + h); + TransposeWxH_32_AVX2(src + (ptrdiff_t)(h - 1) * src_stride, -src_stride, + dst_opt + (ptrdiff_t)(w - 1) * dst_stride, + -dst_stride, w, h); + ASSERT_EQ(0, memcmp(dst_c, dst_opt, dst_bytes)) << "neg " << w << "x" << h; + } + } + + free_aligned_buffer_page_end(src_pixels); + free_aligned_buffer_page_end(dst_pixels_c); + free_aligned_buffer_page_end(dst_pixels_opt); +} +#endif // defined(HAS_TRANSPOSEWXH_32_AVX2) + #endif // ENABLE_ROW_TESTS } // namespace libyuv diff --git a/unit_test/scale_plane_test.cc b/unit_test/scale_plane_test.cc index bf2ec6b8a..012377d5c 100644 --- a/unit_test/scale_plane_test.cc +++ b/unit_test/scale_plane_test.cc @@ -267,6 +267,13 @@ TEST_F(LibYUVScaleTest, TestScaleRowDown2Box_16) { } else { ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280); } +#elif defined(HAS_SCALEROWDOWN2_16_AVX2) + int has_avx2 = TestCpuFlag(kCpuHasAVX2); + if (has_avx2) { + ScaleRowDown2Box_16_AVX2(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280); + } else { + ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280); + } #else ScaleRowDown2Box_16_C(&orig_pixels[0], 2560, &dst_pixels_opt[0], 1280); #endif diff --git a/unit_test/scale_uv_test.cc b/unit_test/scale_uv_test.cc index 5a8d160a9..bbb6bbc68 100644 --- a/unit_test/scale_uv_test.cc +++ b/unit_test/scale_uv_test.cc @@ -39,6 +39,9 @@ static int UVTestFilter(int src_width, if (!SizeValid(src_width, src_height, dst_width, dst_height)) { return 0; } + if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) { + return 0; + } int i; int64_t src_uv_plane_size = Abs(src_width) * Abs(src_height) * 2LL; @@ -259,6 +262,9 @@ static int UVTestBoxAgainstPlanes(int src_width, if (!SizeValid(src_width, src_height, dst_width, dst_height)) { return 0; } + if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) { + return 0; + } int64_t src_uv_plane_size = src_width * src_height * 2LL; int64_t src_plane_size = src_width * src_height * 1LL; int64_t dst_uv_plane_size = dst_width * dst_height * 2LL; @@ -369,4 +375,283 @@ TEST_F(LibYUVScaleTest, UVScaleBoxAverages) { free_aligned_buffer_page_end(src_uv); } +// Test 16-bit UVScale_16 with C vs Opt and return maximum pixel difference. +static int UVTestFilter_16(int src_width, + int src_height, + int dst_width, + int dst_height, + FilterMode f, + int benchmark_iterations, + int disable_cpu_flags, + int benchmark_cpu_info) { + if (!SizeValid(src_width, src_height, dst_width, dst_height)) { + return 0; + } + if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) { + return 0; + } + + int i; + int64_t src_uv_samples = Abs(src_width) * Abs(src_height) * 2LL; + int src_stride_uv = Abs(src_width) * 2; + int64_t dst_uv_samples = dst_width * dst_height * 2LL; + int dst_stride_uv = dst_width * 2; + + align_buffer_page_end(src_uv_buf, src_uv_samples * 2); + align_buffer_page_end(dst_uv_c_buf, dst_uv_samples * 2); + align_buffer_page_end(dst_uv_opt_buf, dst_uv_samples * 2); + + if (!src_uv_buf || !dst_uv_c_buf || !dst_uv_opt_buf) { + printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n"); + return 0; + } + uint16_t* src_uv = reinterpret_cast(src_uv_buf); + uint16_t* dst_uv_c = reinterpret_cast(dst_uv_c_buf); + uint16_t* dst_uv_opt = reinterpret_cast(dst_uv_opt_buf); + + MemRandomize(src_uv_buf, src_uv_samples * 2); + memset(dst_uv_c_buf, 2, dst_uv_samples * 2); + memset(dst_uv_opt_buf, 123, dst_uv_samples * 2); + + MaskCpuFlags(disable_cpu_flags); // Disable all CPU optimization. + int rc_c = UVScale_16(src_uv, src_stride_uv, src_width, src_height, dst_uv_c, + dst_stride_uv, dst_width, dst_height, f); + if (rc_c != 0) { + free_aligned_buffer_page_end(dst_uv_c_buf); + free_aligned_buffer_page_end(dst_uv_opt_buf); + free_aligned_buffer_page_end(src_uv_buf); + return 65536; + } + + MaskCpuFlags(benchmark_cpu_info); // Enable all CPU optimization. + int rc_opt = 0; + for (i = 0; i < benchmark_iterations; ++i) { + rc_opt |= UVScale_16(src_uv, src_stride_uv, src_width, src_height, + dst_uv_opt, dst_stride_uv, dst_width, dst_height, f); + } + if (rc_opt != 0) { + free_aligned_buffer_page_end(dst_uv_c_buf); + free_aligned_buffer_page_end(dst_uv_opt_buf); + free_aligned_buffer_page_end(src_uv_buf); + return 65536; + } + + int max_diff = 0; + for (i = 0; i < dst_uv_samples; ++i) { + int abs_diff = Abs(static_cast(dst_uv_c[i]) - + static_cast(dst_uv_opt[i])); + if (abs_diff > max_diff) { + max_diff = abs_diff; + } + } + + free_aligned_buffer_page_end(dst_uv_c_buf); + free_aligned_buffer_page_end(dst_uv_opt_buf); + free_aligned_buffer_page_end(src_uv_buf); + return max_diff; +} + +#define DX(x, nom, denom) static_cast((Abs(x) / nom) * nom) +#define SX(x, nom, denom) static_cast((x / nom) * denom) + +#define TEST_FACTOR1_16(name, filter, nom, denom) \ + TEST_F(LibYUVScaleTest, UVScale_16_DownBy##name##_##filter) { \ + int diff = UVTestFilter_16( \ + SX(benchmark_width_, nom, denom), SX(benchmark_height_, nom, denom), \ + DX(benchmark_width_, nom, denom), DX(benchmark_height_, nom, denom), \ + kFilter##filter, benchmark_iterations_, disable_cpu_flags_, \ + benchmark_cpu_info_); \ + ASSERT_EQ(0, diff); \ + } + +#if defined(ENABLE_FULL_TESTS) +#define TEST_FACTOR_16(name, nom, denom) \ + TEST_FACTOR1_16(name, None, nom, denom) \ + TEST_FACTOR1_16(name, Linear, nom, denom) \ + TEST_FACTOR1_16(name, Bilinear, nom, denom) \ + TEST_FACTOR1_16(name, Box, nom, denom) +#else +#define TEST_FACTOR_16(name, nom, denom) \ + TEST_FACTOR1_16(name, Bilinear, nom, denom) +#endif + +TEST_FACTOR_16(2, 1, 2) +TEST_FACTOR_16(4, 1, 4) +TEST_FACTOR_16(3by4, 3, 4) +TEST_FACTOR_16(3by8, 3, 8) +TEST_FACTOR_16(3, 1, 3) +#undef TEST_FACTOR1_16 +#undef TEST_FACTOR_16 +#undef SX +#undef DX + +#define TEST_SCALETO1_16(name, width, height, filter, max_diff) \ + TEST_F(LibYUVScaleTest, name##To##width##x##height##_##filter) { \ + int diff = UVTestFilter_16(benchmark_width_, benchmark_height_, width, \ + height, kFilter##filter, benchmark_iterations_, \ + disable_cpu_flags_, benchmark_cpu_info_); \ + ASSERT_LE(diff, max_diff); \ + } \ + TEST_F(LibYUVScaleTest, name##From##width##x##height##_##filter) { \ + int diff = UVTestFilter_16(width, height, Abs(benchmark_width_), \ + Abs(benchmark_height_), kFilter##filter, \ + benchmark_iterations_, disable_cpu_flags_, \ + benchmark_cpu_info_); \ + ASSERT_LE(diff, max_diff); \ + } + +#if defined(ENABLE_FULL_TESTS) +#define TEST_SCALETO_16(name, width, height) \ + TEST_SCALETO1_16(name, width, height, None, 0) \ + TEST_SCALETO1_16(name, width, height, Linear, 0) \ + TEST_SCALETO1_16(name, width, height, Bilinear, 0) +#else +#define TEST_SCALETO_16(name, width, height) \ + TEST_SCALETO1_16(name, width, height, Bilinear, 0) +#endif + +TEST_SCALETO_16(UVScale_16, 1, 1) +TEST_SCALETO_16(UVScale_16, 569, 480) +TEST_SCALETO_16(UVScale_16, 640, 360) +#undef TEST_SCALETO1_16 +#undef TEST_SCALETO_16 + +// Compare UVScale_16 against scaling U and V planes separately with +// ScalePlane_16 across Box, Bilinear, Linear, and None filters. +static int UVTestAgainstPlanes_16(int src_width, + int src_height, + int dst_width, + int dst_height, + FilterMode filtering, + int cpu_info) { + if (!SizeValid(src_width, src_height, dst_width, dst_height)) { + return 0; + } + if (src_width <= 0 || src_height == 0 || dst_width <= 0 || dst_height <= 0) { + return 0; + } + const int abs_src_width = Abs(src_width); + const int abs_src_height = Abs(src_height); + int64_t src_uv_samples = abs_src_width * abs_src_height * 2LL; + int64_t src_plane_samples = abs_src_width * abs_src_height * 1LL; + int64_t dst_uv_samples = dst_width * dst_height * 2LL; + int64_t dst_plane_samples = dst_width * dst_height * 1LL; + + align_buffer_page_end(src_uv_buf, src_uv_samples * 2); + align_buffer_page_end(src_u_buf, src_plane_samples * 2); + align_buffer_page_end(src_v_buf, src_plane_samples * 2); + align_buffer_page_end(dst_uv_buf, dst_uv_samples * 2); + align_buffer_page_end(dst_u_buf, dst_plane_samples * 2); + align_buffer_page_end(dst_v_buf, dst_plane_samples * 2); + if (!src_uv_buf || !src_u_buf || !src_v_buf || !dst_uv_buf || !dst_u_buf || + !dst_v_buf) { + printf("Skipped. Alloc failed " FILELINESTR(__FILE__, __LINE__) "\n"); + return 0; + } + uint16_t* src_uv = reinterpret_cast(src_uv_buf); + uint16_t* src_u = reinterpret_cast(src_u_buf); + uint16_t* src_v = reinterpret_cast(src_v_buf); + uint16_t* dst_uv = reinterpret_cast(dst_uv_buf); + uint16_t* dst_u = reinterpret_cast(dst_u_buf); + uint16_t* dst_v = reinterpret_cast(dst_v_buf); + + MemRandomize(src_uv_buf, src_uv_samples * 2); + memset(dst_uv_buf, 123, dst_uv_samples * 2); + for (int i = 0; i < abs_src_width * abs_src_height; ++i) { + src_u[i] = src_uv[i * 2 + 0]; + src_v[i] = src_uv[i * 2 + 1]; + } + + MaskCpuFlags(cpu_info); + int rc_uv = UVScale_16(src_uv, abs_src_width * 2, src_width, src_height, + dst_uv, dst_width * 2, dst_width, dst_height, + filtering); + int rc_u = ScalePlane_16(src_u, abs_src_width, src_width, src_height, dst_u, + dst_width, dst_width, dst_height, filtering); + int rc_v = ScalePlane_16(src_v, abs_src_width, src_width, src_height, dst_v, + dst_width, dst_width, dst_height, filtering); + int max_diff = (rc_uv != 0 || rc_u != 0 || rc_v != 0) ? 65536 : 0; + for (int i = 0; i < dst_width * dst_height; ++i) { + int du = Abs(static_cast(dst_uv[i * 2 + 0]) - + static_cast(dst_u[i])); + int dv = Abs(static_cast(dst_uv[i * 2 + 1]) - + static_cast(dst_v[i])); + if (du > max_diff) { + max_diff = du; + } + if (dv > max_diff) { + max_diff = dv; + } + } + + free_aligned_buffer_page_end(dst_v_buf); + free_aligned_buffer_page_end(dst_u_buf); + free_aligned_buffer_page_end(dst_uv_buf); + free_aligned_buffer_page_end(src_v_buf); + free_aligned_buffer_page_end(src_u_buf); + free_aligned_buffer_page_end(src_uv_buf); + return max_diff; +} + +#define TEST_UVPLANE16(name, swidth, sheight, dwidth, dheight, filter, \ + max_diff) \ + TEST_F(LibYUVScaleTest, UVScale_16_MatchesPlane_##name##_##filter) { \ + int diff_c = UVTestAgainstPlanes_16(swidth, sheight, dwidth, dheight, \ + kFilter##filter, disable_cpu_flags_); \ + ASSERT_LE(diff_c, max_diff); \ + int diff_opt = UVTestAgainstPlanes_16(swidth, sheight, dwidth, dheight, \ + kFilter##filter, \ + benchmark_cpu_info_); \ + ASSERT_LE(diff_opt, max_diff); \ + } + +TEST_UVPLANE16(By2, 480, 270, 240, 135, Box, 0) +TEST_UVPLANE16(By2, 480, 270, 240, 135, Bilinear, 0) +TEST_UVPLANE16(By2, 480, 270, 240, 135, None, 0) +TEST_UVPLANE16(By3, 480, 270, 160, 90, Box, 0) +TEST_UVPLANE16(By3Rounded, 320, 180, 107, 60, Box, 0) +TEST_UVPLANE16(By4, 480, 270, 120, 68, Box, 0) +TEST_UVPLANE16(By4Exact, 480, 272, 120, 68, Box, 1) +TEST_UVPLANE16(By6, 360, 180, 60, 30, Box, 0) +TEST_UVPLANE16(By6, 360, 180, 60, 30, None, 0) +TEST_UVPLANE16(OddTail, 33, 17, 7, 5, Box, 0) +TEST_UVPLANE16(By3OddTail, 69, 15, 23, 5, Box, 0) +TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, Bilinear, 0) +TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, Linear, 0) +TEST_UVPLANE16(DownArbitrary, 113, 71, 47, 29, None, 0) +TEST_UVPLANE16(Up2x, 37, 23, 74, 46, Bilinear, 0) +TEST_UVPLANE16(Up2x, 37, 23, 74, 46, Linear, 0) +TEST_UVPLANE16(Up2x, 37, 23, 74, 46, None, 0) +TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, Bilinear, 0) +TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, Linear, 0) +TEST_UVPLANE16(UpArbitrary, 37, 23, 91, 59, None, 0) +TEST_UVPLANE16(VerticalOnlyUp, 41, 19, 41, 53, Bilinear, 0) +TEST_UVPLANE16(VerticalOnlyDown, 41, 53, 41, 19, Bilinear, 0) +TEST_UVPLANE16(InvertCopy, 31, -17, 31, 17, None, 0) +TEST_UVPLANE16(InvertUp, 31, -17, 65, 39, Bilinear, 0) +#undef TEST_UVPLANE16 + +// Verify 2x kFilterLinear downscale averages pairs horizontally on odd rows +// without underflowing into the previous row. +TEST_F(LibYUVScaleTest, UVScale_16_DownBy2_LinearAveragesOddRows) { + const int kSrcWidth = 4; + const int kSrcHeight = 2; + const int kDstWidth = 2; + const int kDstHeight = 1; + uint16_t src_uv[kSrcWidth * kSrcHeight * 2] = { + // Row 0 (even row - must not be read): + 60000, 60000, 60000, 60000, 60000, 60000, 60000, 60000, + // Row 1 (odd row - horizontally averaged in pairs): + 100, 1000, 300, 3000, 500, 5000, 900, 7000, + }; + uint16_t dst_uv[kDstWidth * kDstHeight * 2] = {0}; + + ASSERT_EQ(0, UVScale_16(src_uv, kSrcWidth * 2, kSrcWidth, kSrcHeight, dst_uv, + kDstWidth * 2, kDstWidth, kDstHeight, kFilterLinear)); + EXPECT_EQ(200, dst_uv[0]); + EXPECT_EQ(2000, dst_uv[1]); + EXPECT_EQ(700, dst_uv[2]); + EXPECT_EQ(6000, dst_uv[3]); +} + } // namespace libyuv diff --git a/unit_test/unit_test.cc b/unit_test/unit_test.cc index ee1f63e64..c51b184e4 100644 --- a/unit_test/unit_test.cc +++ b/unit_test/unit_test.cc @@ -209,6 +209,12 @@ static int TestCpuEnv(int cpu_info) { if (TestEnv("LIBYUV_DISABLE_AMXINT8")) { cpu_info &= ~libyuv::kCpuHasAMXINT8; } + if (TestEnv("LIBYUV_DISABLE_AVX512BMM")) { + cpu_info &= ~libyuv::kCpuHasAVX512BMM; + } + if (TestEnv("LIBYUV_DISABLE_AMXTF32")) { + cpu_info &= ~libyuv::kCpuHasAMXTF32; + } #endif if (TestEnv("LIBYUV_DISABLE_ASM")) { cpu_info = libyuv::kCpuInitialized; diff --git a/util/cpuid.c b/util/cpuid.c index 33cd4201a..d85595297 100644 --- a/util/cpuid.c +++ b/util/cpuid.c @@ -217,6 +217,7 @@ int main(int argc, const char* argv[]) { int has_avxvnniint8 = TestCpuFlag(kCpuHasAVXVNNIINT8); int has_amxint8 = TestCpuFlag(kCpuHasAMXINT8); int has_avx512bmm = TestCpuFlag(kCpuHasAVX512BMM); + int has_amxtf32 = TestCpuFlag(kCpuHasAMXTF32); printf("Has X86 0x%x\n", has_x86); printf("Has SSE2 0x%x\n", has_sse2); printf("Has SSSE3 0x%x\n", has_ssse3); @@ -240,6 +241,7 @@ int main(int argc, const char* argv[]) { printf("Has AVXVNNIINT8 0x%x\n", has_avxvnniint8); printf("Has AMXINT8 0x%x\n", has_amxint8); printf("Has AVX512BMM 0x%x\n", has_avx512bmm); + printf("Has AMXTF32 0x%x\n", has_amxtf32); #ifdef __linux__ // Test VDPPHPS instruction