diff --git a/include/libyuv/loongson_intrinsics.h b/include/libyuv/loongson_intrinsics.h index 1d613defb..7dd44c575 100644 --- a/include/libyuv/loongson_intrinsics.h +++ b/include/libyuv/loongson_intrinsics.h @@ -8,1942 +8,7 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H -#define INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H +#ifndef INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_ +#define INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_ -/* - * Copyright (c) 2022 Loongson Technology Corporation Limited - * All rights reserved. - * Contributed by Shiyou Yin - * Xiwei Gu - * Lu Wang - * - * This file is a header file for loongarch builtin extension. - * - */ - -#ifndef LOONGSON_INTRINSICS_H -#define LOONGSON_INTRINSICS_H - -/** - * MAJOR version: Macro usage changes. - * MINOR version: Add new functions, or bug fixes. - * MICRO version: Comment changes or implementation changes. - */ -#define LSOM_VERSION_MAJOR 1 -#define LSOM_VERSION_MINOR 1 -#define LSOM_VERSION_MICRO 0 - -#define DUP2_ARG1(_INS, _IN0, _IN1, _OUT0, _OUT1) \ - { \ - _OUT0 = _INS(_IN0); \ - _OUT1 = _INS(_IN1); \ - } - -#define DUP2_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1) \ - { \ - _OUT0 = _INS(_IN0, _IN1); \ - _OUT1 = _INS(_IN2, _IN3); \ - } - -#define DUP2_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1) \ - { \ - _OUT0 = _INS(_IN0, _IN1, _IN2); \ - _OUT1 = _INS(_IN3, _IN4, _IN5); \ - } - -#define DUP4_ARG1(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1, _OUT2, _OUT3) \ - { \ - DUP2_ARG1(_INS, _IN0, _IN1, _OUT0, _OUT1); \ - DUP2_ARG1(_INS, _IN2, _IN3, _OUT2, _OUT3); \ - } - -#define DUP4_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, _OUT0, \ - _OUT1, _OUT2, _OUT3) \ - { \ - DUP2_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1); \ - DUP2_ARG2(_INS, _IN4, _IN5, _IN6, _IN7, _OUT2, _OUT3); \ - } - -#define DUP4_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, _IN8, \ - _IN9, _IN10, _IN11, _OUT0, _OUT1, _OUT2, _OUT3) \ - { \ - DUP2_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1); \ - DUP2_ARG3(_INS, _IN6, _IN7, _IN8, _IN9, _IN10, _IN11, _OUT2, _OUT3); \ - } - -#ifdef __loongarch_sx -#include -/* - * ============================================================================= - * Description : Dot product & addition of byte vector elements - * Arguments : Inputs - in_c, in_h, in_l - * Outputs - out - * Return Type - halfword - * Details : Signed byte elements from in_h are multiplied by - * signed byte elements from in_l, and then added adjacent to - * each other to get results with the twice size of input. - * Then the results plus to signed half-word elements from in_c. - * Example : out = __lsx_vdp2add_h_b(in_c, in_h, in_l) - * in_c : 1,2,3,4, 1,2,3,4 - * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 - * out : 23,40,41,26, 23,40,41,26 - * ============================================================================= - */ -static inline __m128i __lsx_vdp2add_h_b(__m128i in_c, - __m128i in_h, - __m128i in_l) { - __m128i out; - - out = __lsx_vmaddwev_h_b(in_c, in_h, in_l); - out = __lsx_vmaddwod_h_b(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product & addition of byte vector elements - * Arguments : Inputs - in_c, in_h, in_l - * Outputs - out - * Return Type - halfword - * Details : Unsigned byte elements from in_h are multiplied by - * unsigned byte elements from in_l, and then added adjacent to - * each other to get results with the twice size of input. - * The results plus to signed half-word elements from in_c. - * Example : out = __lsx_vdp2add_h_bu(in_c, in_h, in_l) - * in_c : 1,2,3,4, 1,2,3,4 - * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 - * out : 23,40,41,26, 23,40,41,26 - * ============================================================================= - */ -static inline __m128i __lsx_vdp2add_h_bu(__m128i in_c, - __m128i in_h, - __m128i in_l) { - __m128i out; - - out = __lsx_vmaddwev_h_bu(in_c, in_h, in_l); - out = __lsx_vmaddwod_h_bu(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product & addition of byte vector elements - * Arguments : Inputs - in_c, in_h, in_l - * Outputs - out - * Return Type - halfword - * Details : Unsigned byte elements from in_h are multiplied by - * signed byte elements from in_l, and then added adjacent to - * each other to get results with the twice size of input. - * The results plus to signed half-word elements from in_c. - * Example : out = __lsx_vdp2add_h_bu_b(in_c, in_h, in_l) - * in_c : 1,1,1,1, 1,1,1,1 - * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * in_l : -1,-2,-3,-4, -5,-6,-7,-8, 1,2,3,4, 5,6,7,8 - * out : -4,-24,-60,-112, 6,26,62,114 - * ============================================================================= - */ -static inline __m128i __lsx_vdp2add_h_bu_b(__m128i in_c, - __m128i in_h, - __m128i in_l) { - __m128i out; - - out = __lsx_vmaddwev_h_bu_b(in_c, in_h, in_l); - out = __lsx_vmaddwod_h_bu_b(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product & addition of half-word vector elements - * Arguments : Inputs - in_c, in_h, in_l - * Outputs - out - * Return Type - __m128i - * Details : Signed half-word elements from in_h are multiplied by - * signed half-word elements from in_l, and then added adjacent to - * each other to get results with the twice size of input. - * Then the results plus to signed word elements from in_c. - * Example : out = __lsx_vdp2add_h_b(in_c, in_h, in_l) - * in_c : 1,2,3,4 - * in_h : 1,2,3,4, 5,6,7,8 - * in_l : 8,7,6,5, 4,3,2,1 - * out : 23,40,41,26 - * ============================================================================= - */ -static inline __m128i __lsx_vdp2add_w_h(__m128i in_c, - __m128i in_h, - __m128i in_l) { - __m128i out; - - out = __lsx_vmaddwev_w_h(in_c, in_h, in_l); - out = __lsx_vmaddwod_w_h(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of byte vector elements - * Arguments : Inputs - in_h, in_l - * Outputs - out - * Return Type - halfword - * Details : Signed byte elements from in_h are multiplied by - * signed byte elements from in_l, and then added adjacent to - * each other to get results with the twice size of input. - * Example : out = __lsx_vdp2_h_b(in_h, in_l) - * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 - * out : 22,38,38,22, 22,38,38,22 - * ============================================================================= - */ -static inline __m128i __lsx_vdp2_h_b(__m128i in_h, __m128i in_l) { - __m128i out; - - out = __lsx_vmulwev_h_b(in_h, in_l); - out = __lsx_vmaddwod_h_b(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of byte vector elements - * Arguments : Inputs - in_h, in_l - * Outputs - out - * Return Type - halfword - * Details : Unsigned byte elements from in_h are multiplied by - * unsigned byte elements from in_l, and then added adjacent to - * each other to get results with the twice size of input. - * Example : out = __lsx_vdp2_h_bu(in_h, in_l) - * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 - * out : 22,38,38,22, 22,38,38,22 - * ============================================================================= - */ -static inline __m128i __lsx_vdp2_h_bu(__m128i in_h, __m128i in_l) { - __m128i out; - - out = __lsx_vmulwev_h_bu(in_h, in_l); - out = __lsx_vmaddwod_h_bu(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of byte vector elements - * Arguments : Inputs - in_h, in_l - * Outputs - out - * Return Type - halfword - * Details : Unsigned byte elements from in_h are multiplied by - * signed byte elements from in_l, and then added adjacent to - * each other to get results with the twice size of input. - * Example : out = __lsx_vdp2_h_bu_b(in_h, in_l) - * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,-1 - * out : 22,38,38,22, 22,38,38,6 - * ============================================================================= - */ -static inline __m128i __lsx_vdp2_h_bu_b(__m128i in_h, __m128i in_l) { - __m128i out; - - out = __lsx_vmulwev_h_bu_b(in_h, in_l); - out = __lsx_vmaddwod_h_bu_b(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of byte vector elements - * Arguments : Inputs - in_h, in_l - * Outputs - out - * Return Type - halfword - * Details : Signed byte elements from in_h are multiplied by - * signed byte elements from in_l, and then added adjacent to - * each other to get results with the twice size of input. - * Example : out = __lsx_vdp2_w_h(in_h, in_l) - * in_h : 1,2,3,4, 5,6,7,8 - * in_l : 8,7,6,5, 4,3,2,1 - * out : 22,38,38,22 - * ============================================================================= - */ -static inline __m128i __lsx_vdp2_w_h(__m128i in_h, __m128i in_l) { - __m128i out; - - out = __lsx_vmulwev_w_h(in_h, in_l); - out = __lsx_vmaddwod_w_h(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Clip all halfword elements of input vector between min & max - * out = ((_in) < (min)) ? (min) : (((_in) > (max)) ? (max) : - * (_in)) - * Arguments : Inputs - _in (input vector) - * - min (min threshold) - * - max (max threshold) - * Outputs - out (output vector with clipped elements) - * Return Type - signed halfword - * Example : out = __lsx_vclip_h(_in) - * _in : -8,2,280,249, -8,255,280,249 - * min : 1,1,1,1, 1,1,1,1 - * max : 9,9,9,9, 9,9,9,9 - * out : 1,2,9,9, 1,9,9,9 - * ============================================================================= - */ -static inline __m128i __lsx_vclip_h(__m128i _in, __m128i min, __m128i max) { - __m128i out; - - out = __lsx_vmax_h(min, _in); - out = __lsx_vmin_h(max, out); - return out; -} - -/* - * ============================================================================= - * Description : Set each element of vector between 0 and 255 - * Arguments : Inputs - _in - * Outputs - out - * Return Type - halfword - * Details : Signed byte elements from _in are clamped between 0 and 255. - * Example : out = __lsx_vclip255_h(_in) - * _in : -8,255,280,249, -8,255,280,249 - * out : 0,255,255,249, 0,255,255,249 - * ============================================================================= - */ -static inline __m128i __lsx_vclip255_h(__m128i _in) { - __m128i out; - - out = __lsx_vmaxi_h(_in, 0); - out = __lsx_vsat_hu(out, 7); - return out; -} - -/* - * ============================================================================= - * Description : Set each element of vector between 0 and 255 - * Arguments : Inputs - _in - * Outputs - out - * Return Type - word - * Details : Signed byte elements from _in are clamped between 0 and 255. - * Example : out = __lsx_vclip255_w(_in) - * _in : -8,255,280,249 - * out : 0,255,255,249 - * ============================================================================= - */ -static inline __m128i __lsx_vclip255_w(__m128i _in) { - __m128i out; - - out = __lsx_vmaxi_w(_in, 0); - out = __lsx_vsat_wu(out, 7); - return out; -} - -/* - * ============================================================================= - * Description : Swap two variables - * Arguments : Inputs - _in0, _in1 - * Outputs - _in0, _in1 (in-place) - * Details : Swapping of two input variables using xor - * Example : LSX_SWAP(_in0, _in1) - * _in0 : 1,2,3,4 - * _in1 : 5,6,7,8 - * _in0(out) : 5,6,7,8 - * _in1(out) : 1,2,3,4 - * ============================================================================= - */ -#define LSX_SWAP(_in0, _in1) \ - { \ - _in0 = __lsx_vxor_v(_in0, _in1); \ - _in1 = __lsx_vxor_v(_in0, _in1); \ - _in0 = __lsx_vxor_v(_in0, _in1); \ - } - -/* - * ============================================================================= - * Description : Transpose 4x4 block with word elements in vectors - * Arguments : Inputs - in0, in1, in2, in3 - * Outputs - out0, out1, out2, out3 - * Details : - * Example : - * 1, 2, 3, 4 1, 5, 9,13 - * 5, 6, 7, 8 to 2, 6,10,14 - * 9,10,11,12 =====> 3, 7,11,15 - * 13,14,15,16 4, 8,12,16 - * ============================================================================= - */ -#define LSX_TRANSPOSE4x4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - __m128i _t0, _t1, _t2, _t3; \ - \ - _t0 = __lsx_vilvl_w(_in1, _in0); \ - _t1 = __lsx_vilvh_w(_in1, _in0); \ - _t2 = __lsx_vilvl_w(_in3, _in2); \ - _t3 = __lsx_vilvh_w(_in3, _in2); \ - _out0 = __lsx_vilvl_d(_t2, _t0); \ - _out1 = __lsx_vilvh_d(_t2, _t0); \ - _out2 = __lsx_vilvl_d(_t3, _t1); \ - _out3 = __lsx_vilvh_d(_t3, _t1); \ - } - -/* - * ============================================================================= - * Description : Transpose 8x8 block with byte elements in vectors - * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 - * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, - * _out7 - * Details : The rows of the matrix become columns, and the columns - * become rows. - * Example : LSX_TRANSPOSE8x8_B - * _in0 : 00,01,02,03,04,05,06,07, 00,00,00,00,00,00,00,00 - * _in1 : 10,11,12,13,14,15,16,17, 00,00,00,00,00,00,00,00 - * _in2 : 20,21,22,23,24,25,26,27, 00,00,00,00,00,00,00,00 - * _in3 : 30,31,32,33,34,35,36,37, 00,00,00,00,00,00,00,00 - * _in4 : 40,41,42,43,44,45,46,47, 00,00,00,00,00,00,00,00 - * _in5 : 50,51,52,53,54,55,56,57, 00,00,00,00,00,00,00,00 - * _in6 : 60,61,62,63,64,65,66,67, 00,00,00,00,00,00,00,00 - * _in7 : 70,71,72,73,74,75,76,77, 00,00,00,00,00,00,00,00 - * - * _ out0 : 00,10,20,30,40,50,60,70, 00,00,00,00,00,00,00,00 - * _ out1 : 01,11,21,31,41,51,61,71, 00,00,00,00,00,00,00,00 - * _ out2 : 02,12,22,32,42,52,62,72, 00,00,00,00,00,00,00,00 - * _ out3 : 03,13,23,33,43,53,63,73, 00,00,00,00,00,00,00,00 - * _ out4 : 04,14,24,34,44,54,64,74, 00,00,00,00,00,00,00,00 - * _ out5 : 05,15,25,35,45,55,65,75, 00,00,00,00,00,00,00,00 - * _ out6 : 06,16,26,36,46,56,66,76, 00,00,00,00,00,00,00,00 - * _ out7 : 07,17,27,37,47,57,67,77, 00,00,00,00,00,00,00,00 - * ============================================================================= - */ -#define LSX_TRANSPOSE8x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - __m128i zero = {0}; \ - __m128i shuf8 = {0x0F0E0D0C0B0A0908, 0x1716151413121110}; \ - __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ - \ - _t0 = __lsx_vilvl_b(_in2, _in0); \ - _t1 = __lsx_vilvl_b(_in3, _in1); \ - _t2 = __lsx_vilvl_b(_in6, _in4); \ - _t3 = __lsx_vilvl_b(_in7, _in5); \ - _t4 = __lsx_vilvl_b(_t1, _t0); \ - _t5 = __lsx_vilvh_b(_t1, _t0); \ - _t6 = __lsx_vilvl_b(_t3, _t2); \ - _t7 = __lsx_vilvh_b(_t3, _t2); \ - _out0 = __lsx_vilvl_w(_t6, _t4); \ - _out2 = __lsx_vilvh_w(_t6, _t4); \ - _out4 = __lsx_vilvl_w(_t7, _t5); \ - _out6 = __lsx_vilvh_w(_t7, _t5); \ - _out1 = __lsx_vshuf_b(zero, _out0, shuf8); \ - _out3 = __lsx_vshuf_b(zero, _out2, shuf8); \ - _out5 = __lsx_vshuf_b(zero, _out4, shuf8); \ - _out7 = __lsx_vshuf_b(zero, _out6, shuf8); \ - } - -/* - * ============================================================================= - * Description : Transpose 8x8 block with half-word elements in vectors - * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7 - * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 - * Details : - * Example : - * 00,01,02,03,04,05,06,07 00,10,20,30,40,50,60,70 - * 10,11,12,13,14,15,16,17 01,11,21,31,41,51,61,71 - * 20,21,22,23,24,25,26,27 02,12,22,32,42,52,62,72 - * 30,31,32,33,34,35,36,37 to 03,13,23,33,43,53,63,73 - * 40,41,42,43,44,45,46,47 ======> 04,14,24,34,44,54,64,74 - * 50,51,52,53,54,55,56,57 05,15,25,35,45,55,65,75 - * 60,61,62,63,64,65,66,67 06,16,26,36,46,56,66,76 - * 70,71,72,73,74,75,76,77 07,17,27,37,47,57,67,77 - * ============================================================================= - */ -#define LSX_TRANSPOSE8x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - __m128i _s0, _s1, _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ - \ - _s0 = __lsx_vilvl_h(_in6, _in4); \ - _s1 = __lsx_vilvl_h(_in7, _in5); \ - _t0 = __lsx_vilvl_h(_s1, _s0); \ - _t1 = __lsx_vilvh_h(_s1, _s0); \ - _s0 = __lsx_vilvh_h(_in6, _in4); \ - _s1 = __lsx_vilvh_h(_in7, _in5); \ - _t2 = __lsx_vilvl_h(_s1, _s0); \ - _t3 = __lsx_vilvh_h(_s1, _s0); \ - _s0 = __lsx_vilvl_h(_in2, _in0); \ - _s1 = __lsx_vilvl_h(_in3, _in1); \ - _t4 = __lsx_vilvl_h(_s1, _s0); \ - _t5 = __lsx_vilvh_h(_s1, _s0); \ - _s0 = __lsx_vilvh_h(_in2, _in0); \ - _s1 = __lsx_vilvh_h(_in3, _in1); \ - _t6 = __lsx_vilvl_h(_s1, _s0); \ - _t7 = __lsx_vilvh_h(_s1, _s0); \ - \ - _out0 = __lsx_vpickev_d(_t0, _t4); \ - _out2 = __lsx_vpickev_d(_t1, _t5); \ - _out4 = __lsx_vpickev_d(_t2, _t6); \ - _out6 = __lsx_vpickev_d(_t3, _t7); \ - _out1 = __lsx_vpickod_d(_t0, _t4); \ - _out3 = __lsx_vpickod_d(_t1, _t5); \ - _out5 = __lsx_vpickod_d(_t2, _t6); \ - _out7 = __lsx_vpickod_d(_t3, _t7); \ - } - -/* - * ============================================================================= - * Description : Transpose input 8x4 byte block into 4x8 - * Arguments : Inputs - _in0, _in1, _in2, _in3 (input 8x4 byte block) - * Outputs - _out0, _out1, _out2, _out3 (output 4x8 byte block) - * Return Type - as per RTYPE - * Details : The rows of the matrix become columns, and the columns become - * rows. - * Example : LSX_TRANSPOSE8x4_B - * _in0 : 00,01,02,03,00,00,00,00, 00,00,00,00,00,00,00,00 - * _in1 : 10,11,12,13,00,00,00,00, 00,00,00,00,00,00,00,00 - * _in2 : 20,21,22,23,00,00,00,00, 00,00,00,00,00,00,00,00 - * _in3 : 30,31,32,33,00,00,00,00, 00,00,00,00,00,00,00,00 - * _in4 : 40,41,42,43,00,00,00,00, 00,00,00,00,00,00,00,00 - * _in5 : 50,51,52,53,00,00,00,00, 00,00,00,00,00,00,00,00 - * _in6 : 60,61,62,63,00,00,00,00, 00,00,00,00,00,00,00,00 - * _in7 : 70,71,72,73,00,00,00,00, 00,00,00,00,00,00,00,00 - * - * _out0 : 00,10,20,30,40,50,60,70, 00,00,00,00,00,00,00,00 - * _out1 : 01,11,21,31,41,51,61,71, 00,00,00,00,00,00,00,00 - * _out2 : 02,12,22,32,42,52,62,72, 00,00,00,00,00,00,00,00 - * _out3 : 03,13,23,33,43,53,63,73, 00,00,00,00,00,00,00,00 - * ============================================================================= - */ -#define LSX_TRANSPOSE8x4_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3) \ - { \ - __m128i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ - \ - _tmp0_m = __lsx_vpackev_w(_in4, _in0); \ - _tmp1_m = __lsx_vpackev_w(_in5, _in1); \ - _tmp2_m = __lsx_vilvl_b(_tmp1_m, _tmp0_m); \ - _tmp0_m = __lsx_vpackev_w(_in6, _in2); \ - _tmp1_m = __lsx_vpackev_w(_in7, _in3); \ - \ - _tmp3_m = __lsx_vilvl_b(_tmp1_m, _tmp0_m); \ - _tmp0_m = __lsx_vilvl_h(_tmp3_m, _tmp2_m); \ - _tmp1_m = __lsx_vilvh_h(_tmp3_m, _tmp2_m); \ - \ - _out0 = __lsx_vilvl_w(_tmp1_m, _tmp0_m); \ - _out2 = __lsx_vilvh_w(_tmp1_m, _tmp0_m); \ - _out1 = __lsx_vilvh_d(_out2, _out0); \ - _out3 = __lsx_vilvh_d(_out0, _out2); \ - } - -/* - * ============================================================================= - * Description : Transpose 16x8 block with byte elements in vectors - * Arguments : Inputs - in0, in1, in2, in3, in4, in5, in6, in7, in8 - * in9, in10, in11, in12, in13, in14, in15 - * Outputs - out0, out1, out2, out3, out4, out5, out6, out7 - * Details : - * Example : - * 000,001,002,003,004,005,006,007 - * 008,009,010,011,012,013,014,015 - * 016,017,018,019,020,021,022,023 - * 024,025,026,027,028,029,030,031 - * 032,033,034,035,036,037,038,039 - * 040,041,042,043,044,045,046,047 000,008,...,112,120 - * 048,049,050,051,052,053,054,055 001,009,...,113,121 - * 056,057,058,059,060,061,062,063 to 002,010,...,114,122 - * 064,068,066,067,068,069,070,071 =====> 003,011,...,115,123 - * 072,073,074,075,076,077,078,079 004,012,...,116,124 - * 080,081,082,083,084,085,086,087 005,013,...,117,125 - * 088,089,090,091,092,093,094,095 006,014,...,118,126 - * 096,097,098,099,100,101,102,103 007,015,...,119,127 - * 104,105,106,107,108,109,110,111 - * 112,113,114,115,116,117,118,119 - * 120,121,122,123,124,125,126,127 - * ============================================================================= - */ -#define LSX_TRANSPOSE16x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ - _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ - _out6, _out7) \ - { \ - __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5, _tmp6, _tmp7; \ - __m128i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ - DUP4_ARG2(__lsx_vilvl_b, _in2, _in0, _in3, _in1, _in6, _in4, _in7, _in5, \ - _tmp0, _tmp1, _tmp2, _tmp3); \ - DUP4_ARG2(__lsx_vilvl_b, _in10, _in8, _in11, _in9, _in14, _in12, _in15, \ - _in13, _tmp4, _tmp5, _tmp6, _tmp7); \ - DUP2_ARG2(__lsx_vilvl_b, _tmp1, _tmp0, _tmp3, _tmp2, _t0, _t2); \ - DUP2_ARG2(__lsx_vilvh_b, _tmp1, _tmp0, _tmp3, _tmp2, _t1, _t3); \ - DUP2_ARG2(__lsx_vilvl_b, _tmp5, _tmp4, _tmp7, _tmp6, _t4, _t6); \ - DUP2_ARG2(__lsx_vilvh_b, _tmp5, _tmp4, _tmp7, _tmp6, _t5, _t7); \ - DUP2_ARG2(__lsx_vilvl_w, _t2, _t0, _t3, _t1, _tmp0, _tmp4); \ - DUP2_ARG2(__lsx_vilvh_w, _t2, _t0, _t3, _t1, _tmp2, _tmp6); \ - DUP2_ARG2(__lsx_vilvl_w, _t6, _t4, _t7, _t5, _tmp1, _tmp5); \ - DUP2_ARG2(__lsx_vilvh_w, _t6, _t4, _t7, _t5, _tmp3, _tmp7); \ - DUP2_ARG2(__lsx_vilvl_d, _tmp1, _tmp0, _tmp3, _tmp2, _out0, _out2); \ - DUP2_ARG2(__lsx_vilvh_d, _tmp1, _tmp0, _tmp3, _tmp2, _out1, _out3); \ - DUP2_ARG2(__lsx_vilvl_d, _tmp5, _tmp4, _tmp7, _tmp6, _out4, _out6); \ - DUP2_ARG2(__lsx_vilvh_d, _tmp5, _tmp4, _tmp7, _tmp6, _out5, _out7); \ - } - -/* - * ============================================================================= - * Description : Butterfly of 4 input vectors - * Arguments : Inputs - in0, in1, in2, in3 - * Outputs - out0, out1, out2, out3 - * Details : Butterfly operation - * Example : - * out0 = in0 + in3; - * out1 = in1 + in2; - * out2 = in1 - in2; - * out3 = in0 - in3; - * ============================================================================= - */ -#define LSX_BUTTERFLY_4_B(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - _out0 = __lsx_vadd_b(_in0, _in3); \ - _out1 = __lsx_vadd_b(_in1, _in2); \ - _out2 = __lsx_vsub_b(_in1, _in2); \ - _out3 = __lsx_vsub_b(_in0, _in3); \ - } -#define LSX_BUTTERFLY_4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - _out0 = __lsx_vadd_h(_in0, _in3); \ - _out1 = __lsx_vadd_h(_in1, _in2); \ - _out2 = __lsx_vsub_h(_in1, _in2); \ - _out3 = __lsx_vsub_h(_in0, _in3); \ - } -#define LSX_BUTTERFLY_4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - _out0 = __lsx_vadd_w(_in0, _in3); \ - _out1 = __lsx_vadd_w(_in1, _in2); \ - _out2 = __lsx_vsub_w(_in1, _in2); \ - _out3 = __lsx_vsub_w(_in0, _in3); \ - } -#define LSX_BUTTERFLY_4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - _out0 = __lsx_vadd_d(_in0, _in3); \ - _out1 = __lsx_vadd_d(_in1, _in2); \ - _out2 = __lsx_vsub_d(_in1, _in2); \ - _out3 = __lsx_vsub_d(_in0, _in3); \ - } - -/* - * ============================================================================= - * Description : Butterfly of 8 input vectors - * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ - * Outputs - _out0, _out1, _out2, _out3, ~ - * Details : Butterfly operation - * Example : - * _out0 = _in0 + _in7; - * _out1 = _in1 + _in6; - * _out2 = _in2 + _in5; - * _out3 = _in3 + _in4; - * _out4 = _in3 - _in4; - * _out5 = _in2 - _in5; - * _out6 = _in1 - _in6; - * _out7 = _in0 - _in7; - * ============================================================================= - */ -#define LSX_BUTTERFLY_8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - _out0 = __lsx_vadd_b(_in0, _in7); \ - _out1 = __lsx_vadd_b(_in1, _in6); \ - _out2 = __lsx_vadd_b(_in2, _in5); \ - _out3 = __lsx_vadd_b(_in3, _in4); \ - _out4 = __lsx_vsub_b(_in3, _in4); \ - _out5 = __lsx_vsub_b(_in2, _in5); \ - _out6 = __lsx_vsub_b(_in1, _in6); \ - _out7 = __lsx_vsub_b(_in0, _in7); \ - } - -#define LSX_BUTTERFLY_8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - _out0 = __lsx_vadd_h(_in0, _in7); \ - _out1 = __lsx_vadd_h(_in1, _in6); \ - _out2 = __lsx_vadd_h(_in2, _in5); \ - _out3 = __lsx_vadd_h(_in3, _in4); \ - _out4 = __lsx_vsub_h(_in3, _in4); \ - _out5 = __lsx_vsub_h(_in2, _in5); \ - _out6 = __lsx_vsub_h(_in1, _in6); \ - _out7 = __lsx_vsub_h(_in0, _in7); \ - } - -#define LSX_BUTTERFLY_8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - _out0 = __lsx_vadd_w(_in0, _in7); \ - _out1 = __lsx_vadd_w(_in1, _in6); \ - _out2 = __lsx_vadd_w(_in2, _in5); \ - _out3 = __lsx_vadd_w(_in3, _in4); \ - _out4 = __lsx_vsub_w(_in3, _in4); \ - _out5 = __lsx_vsub_w(_in2, _in5); \ - _out6 = __lsx_vsub_w(_in1, _in6); \ - _out7 = __lsx_vsub_w(_in0, _in7); \ - } - -#define LSX_BUTTERFLY_8_D(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - _out0 = __lsx_vadd_d(_in0, _in7); \ - _out1 = __lsx_vadd_d(_in1, _in6); \ - _out2 = __lsx_vadd_d(_in2, _in5); \ - _out3 = __lsx_vadd_d(_in3, _in4); \ - _out4 = __lsx_vsub_d(_in3, _in4); \ - _out5 = __lsx_vsub_d(_in2, _in5); \ - _out6 = __lsx_vsub_d(_in1, _in6); \ - _out7 = __lsx_vsub_d(_in0, _in7); \ - } - -#endif // LSX - -#ifdef __loongarch_asx -#include -/* - * ============================================================================= - * Description : Dot product of byte vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - signed halfword - * Details : Unsigned byte elements from in_h are multiplied with - * unsigned byte elements from in_l producing a result - * twice the size of input i.e. signed halfword. - * Then this multiplied results of adjacent odd-even elements - * are added to the out vector - * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2_h_bu(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvmulwev_h_bu(in_h, in_l); - out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of byte vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - signed halfword - * Details : Signed byte elements from in_h are multiplied with - * signed byte elements from in_l producing a result - * twice the size of input i.e. signed halfword. - * Then this multiplication results of adjacent odd-even elements - * are added to the out vector - * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2_h_b(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvmulwev_h_b(in_h, in_l); - out = __lasx_xvmaddwod_h_b(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of halfword vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - signed word - * Details : Signed halfword elements from in_h are multiplied with - * signed halfword elements from in_l producing a result - * twice the size of input i.e. signed word. - * Then this multiplied results of adjacent odd-even elements - * are added to the out vector. - * Example : out = __lasx_xvdp2_w_h(in_h, in_l) - * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1 - * out : 22,38,38,22, 22,38,38,22 - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2_w_h(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvmulwev_w_h(in_h, in_l); - out = __lasx_xvmaddwod_w_h(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of word vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - signed double - * Details : Signed word elements from in_h are multiplied with - * signed word elements from in_l producing a result - * twice the size of input i.e. signed double-word. - * Then this multiplied results of adjacent odd-even elements - * are added to the out vector. - * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2_d_w(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvmulwev_d_w(in_h, in_l); - out = __lasx_xvmaddwod_d_w(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of halfword vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - signed word - * Details : Unsigned halfword elements from in_h are multiplied with - * signed halfword elements from in_l producing a result - * twice the size of input i.e. unsigned word. - * Multiplication result of adjacent odd-even elements - * are added to the out vector - * Example : See out = __lasx_xvdp2_w_h(in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2_w_hu_h(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvmulwev_w_hu_h(in_h, in_l); - out = __lasx_xvmaddwod_w_hu_h(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product & addition of byte vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - halfword - * Details : Signed byte elements from in_h are multiplied with - * signed byte elements from in_l producing a result - * twice the size of input i.e. signed halfword. - * Then this multiplied results of adjacent odd-even elements - * are added to the in_c vector. - * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2add_h_b(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i out; - - out = __lasx_xvmaddwev_h_b(in_c, in_h, in_l); - out = __lasx_xvmaddwod_h_b(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product & addition of byte vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - halfword - * Details : Unsigned byte elements from in_h are multiplied with - * unsigned byte elements from in_l producing a result - * twice the size of input i.e. signed halfword. - * Then this multiplied results of adjacent odd-even elements - * are added to the in_c vector. - * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2add_h_bu(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i out; - - out = __lasx_xvmaddwev_h_bu(in_c, in_h, in_l); - out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product & addition of byte vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - halfword - * Details : Unsigned byte elements from in_h are multiplied with - * signed byte elements from in_l producing a result - * twice the size of input i.e. signed halfword. - * Then this multiplied results of adjacent odd-even elements - * are added to the in_c vector. - * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2add_h_bu_b(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i out; - - out = __lasx_xvmaddwev_h_bu_b(in_c, in_h, in_l); - out = __lasx_xvmaddwod_h_bu_b(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of halfword vector elements - * Arguments : Inputs - in_c, in_h, in_l - * Output - out - * Return Type - per RTYPE - * Details : Signed halfword elements from in_h are multiplied with - * signed halfword elements from in_l producing a result - * twice the size of input i.e. signed word. - * Multiplication result of adjacent odd-even elements - * are added to the in_c vector. - * Example : out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) - * in_c : 1,2,3,4, 1,2,3,4 - * in_h : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8, - * in_l : 8,7,6,5, 4,3,2,1, 8,7,6,5, 4,3,2,1, - * out : 23,40,41,26, 23,40,41,26 - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2add_w_h(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i out; - - out = __lasx_xvmaddwev_w_h(in_c, in_h, in_l); - out = __lasx_xvmaddwod_w_h(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of halfword vector elements - * Arguments : Inputs - in_c, in_h, in_l - * Output - out - * Return Type - signed word - * Details : Unsigned halfword elements from in_h are multiplied with - * unsigned halfword elements from in_l producing a result - * twice the size of input i.e. signed word. - * Multiplication result of adjacent odd-even elements - * are added to the in_c vector. - * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2add_w_hu(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i out; - - out = __lasx_xvmaddwev_w_hu(in_c, in_h, in_l); - out = __lasx_xvmaddwod_w_hu(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of halfword vector elements - * Arguments : Inputs - in_c, in_h, in_l - * Output - out - * Return Type - signed word - * Details : Unsigned halfword elements from in_h are multiplied with - * signed halfword elements from in_l producing a result - * twice the size of input i.e. signed word. - * Multiplication result of adjacent odd-even elements - * are added to the in_c vector - * Example : See out = __lasx_xvdp2add_w_h(in_c, in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2add_w_hu_h(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i out; - - out = __lasx_xvmaddwev_w_hu_h(in_c, in_h, in_l); - out = __lasx_xvmaddwod_w_hu_h(out, in_h, in_l); - return out; -} - -/* - * ============================================================================= - * Description : Vector Unsigned Dot Product and Subtract - * Arguments : Inputs - in_c, in_h, in_l - * Output - out - * Return Type - signed halfword - * Details : Unsigned byte elements from in_h are multiplied with - * unsigned byte elements from in_l producing a result - * twice the size of input i.e. signed halfword. - * Multiplication result of adjacent odd-even elements - * are added together and subtracted from double width elements - * in_c vector. - * Example : See out = __lasx_xvdp2sub_w_h(in_c, in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2sub_h_bu(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i out; - - out = __lasx_xvmulwev_h_bu(in_h, in_l); - out = __lasx_xvmaddwod_h_bu(out, in_h, in_l); - out = __lasx_xvsub_h(in_c, out); - return out; -} - -/* - * ============================================================================= - * Description : Vector Signed Dot Product and Subtract - * Arguments : Inputs - in_c, in_h, in_l - * Output - out - * Return Type - signed word - * Details : Signed halfword elements from in_h are multiplied with - * Signed halfword elements from in_l producing a result - * twice the size of input i.e. signed word. - * Multiplication result of adjacent odd-even elements - * are added together and subtracted from double width elements - * in_c vector. - * Example : out = __lasx_xvdp2sub_w_h(in_c, in_h, in_l) - * in_c : 0,0,0,0, 0,0,0,0 - * in_h : 3,1,3,0, 0,0,0,1, 0,0,1,1, 0,0,0,1 - * in_l : 2,1,1,0, 1,0,0,0, 0,0,1,0, 1,0,0,1 - * out : -7,-3,0,0, 0,-1,0,-1 - * ============================================================================= - */ -static inline __m256i __lasx_xvdp2sub_w_h(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i out; - - out = __lasx_xvmulwev_w_h(in_h, in_l); - out = __lasx_xvmaddwod_w_h(out, in_h, in_l); - out = __lasx_xvsub_w(in_c, out); - return out; -} - -/* - * ============================================================================= - * Description : Dot product of halfword vector elements - * Arguments : Inputs - in_h, in_l - * Output - out - * Return Type - signed word - * Details : Signed halfword elements from in_h are multiplied with - * signed halfword elements from in_l producing a result - * four times the size of input i.e. signed doubleword. - * Then this multiplication results of four adjacent elements - * are added together and stored to the out vector. - * Example : out = __lasx_xvdp4_d_h(in_h, in_l) - * in_h : 3,1,3,0, 0,0,0,1, 0,0,1,-1, 0,0,0,1 - * in_l : -2,1,1,0, 1,0,0,0, 0,0,1, 0, 1,0,0,1 - * out : -2,0,1,1 - * ============================================================================= - */ -static inline __m256i __lasx_xvdp4_d_h(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvmulwev_w_h(in_h, in_l); - out = __lasx_xvmaddwod_w_h(out, in_h, in_l); - out = __lasx_xvhaddw_d_w(out, out); - return out; -} - -/* - * ============================================================================= - * Description : The high half of the vector elements are expanded and - * added after being doubled. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_h vector and the in_l vector are added after the - * higher half of the two-fold sign extension (signed byte - * to signed halfword) and stored to the out vector. - * Example : See out = __lasx_xvaddwh_w_h(in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvaddwh_h_b(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvilvh_b(in_h, in_l); - out = __lasx_xvhaddw_h_b(out, out); - return out; -} - -/* - * ============================================================================= - * Description : The high half of the vector elements are expanded and - * added after being doubled. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_h vector and the in_l vector are added after the - * higher half of the two-fold sign extension (signed halfword - * to signed word) and stored to the out vector. - * Example : out = __lasx_xvaddwh_w_h(in_h, in_l) - * in_h : 3, 0,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 - * in_l : 2,-1,1,2, 1,0,0, 0, 1,0,1, 0, 1,0,0,1 - * out : 1,0,0,-1, 1,0,0, 2 - * ============================================================================= - */ -static inline __m256i __lasx_xvaddwh_w_h(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvilvh_h(in_h, in_l); - out = __lasx_xvhaddw_w_h(out, out); - return out; -} - -/* - * ============================================================================= - * Description : The low half of the vector elements are expanded and - * added after being doubled. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_h vector and the in_l vector are added after the - * lower half of the two-fold sign extension (signed byte - * to signed halfword) and stored to the out vector. - * Example : See out = __lasx_xvaddwl_w_h(in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvaddwl_h_b(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvilvl_b(in_h, in_l); - out = __lasx_xvhaddw_h_b(out, out); - return out; -} - -/* - * ============================================================================= - * Description : The low half of the vector elements are expanded and - * added after being doubled. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_h vector and the in_l vector are added after the - * lower half of the two-fold sign extension (signed halfword - * to signed word) and stored to the out vector. - * Example : out = __lasx_xvaddwl_w_h(in_h, in_l) - * in_h : 3, 0,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 - * in_l : 2,-1,1,2, 1,0,0, 0, 1,0,1, 0, 1,0,0,1 - * out : 5,-1,4,2, 1,0,2,-1 - * ============================================================================= - */ -static inline __m256i __lasx_xvaddwl_w_h(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvilvl_h(in_h, in_l); - out = __lasx_xvhaddw_w_h(out, out); - return out; -} - -/* - * ============================================================================= - * Description : The low half of the vector elements are expanded and - * added after being doubled. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The out vector and the out vector are added after the - * lower half of the two-fold zero extension (unsigned byte - * to unsigned halfword) and stored to the out vector. - * Example : See out = __lasx_xvaddwl_w_h(in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvaddwl_h_bu(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvilvl_b(in_h, in_l); - out = __lasx_xvhaddw_hu_bu(out, out); - return out; -} - -/* - * ============================================================================= - * Description : The low half of the vector elements are expanded and - * added after being doubled. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_l vector after double zero extension (unsigned byte to - * signed halfword)，added to the in_h vector. - * Example : See out = __lasx_xvaddw_w_w_h(in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvaddw_h_h_bu(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvsllwil_hu_bu(in_l, 0); - out = __lasx_xvadd_h(in_h, out); - return out; -} - -/* - * ============================================================================= - * Description : The low half of the vector elements are expanded and - * added after being doubled. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_l vector after double sign extension (signed halfword to - * signed word), added to the in_h vector. - * Example : out = __lasx_xvaddw_w_w_h(in_h, in_l) - * in_h : 0, 1,0,0, -1,0,0,1, - * in_l : 2,-1,1,2, 1,0,0,0, 0,0,1,0, 1,0,0,1, - * out : 2, 0,1,2, -1,0,1,1, - * ============================================================================= - */ -static inline __m256i __lasx_xvaddw_w_w_h(__m256i in_h, __m256i in_l) { - __m256i out; - - out = __lasx_xvsllwil_w_h(in_l, 0); - out = __lasx_xvadd_w(in_h, out); - return out; -} - -/* - * ============================================================================= - * Description : Multiplication and addition calculation after expansion - * of the lower half of the vector. - * Arguments : Inputs - in_c, in_h, in_l - * Output - out - * Details : The in_h vector and the in_l vector are multiplied after - * the lower half of the two-fold sign extension (signed halfword - * to signed word), and the result is added to the vector in_c, - * then stored to the out vector. - * Example : out = __lasx_xvmaddwl_w_h(in_c, in_h, in_l) - * in_c : 1,2,3,4, 5,6,7,8 - * in_h : 1,2,3,4, 1,2,3,4, 5,6,7,8, 5,6,7,8 - * in_l : 200, 300, 400, 500, 2000, 3000, 4000, 5000, - * -200,-300,-400,-500, -2000,-3000,-4000,-5000 - * out : 201, 602,1203,2004, -995, -1794,-2793,-3992 - * ============================================================================= - */ -static inline __m256i __lasx_xvmaddwl_w_h(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i tmp0, tmp1, out; - - tmp0 = __lasx_xvsllwil_w_h(in_h, 0); - tmp1 = __lasx_xvsllwil_w_h(in_l, 0); - tmp0 = __lasx_xvmul_w(tmp0, tmp1); - out = __lasx_xvadd_w(tmp0, in_c); - return out; -} - -/* - * ============================================================================= - * Description : Multiplication and addition calculation after expansion - * of the higher half of the vector. - * Arguments : Inputs - in_c, in_h, in_l - * Output - out - * Details : The in_h vector and the in_l vector are multiplied after - * the higher half of the two-fold sign extension (signed - * halfword to signed word), and the result is added to - * the vector in_c, then stored to the out vector. - * Example : See out = __lasx_xvmaddwl_w_h(in_c, in_h, in_l) - * ============================================================================= - */ -static inline __m256i __lasx_xvmaddwh_w_h(__m256i in_c, - __m256i in_h, - __m256i in_l) { - __m256i tmp0, tmp1, out; - - tmp0 = __lasx_xvilvh_h(in_h, in_h); - tmp1 = __lasx_xvilvh_h(in_l, in_l); - tmp0 = __lasx_xvmulwev_w_h(tmp0, tmp1); - out = __lasx_xvadd_w(tmp0, in_c); - return out; -} - -/* - * ============================================================================= - * Description : Multiplication calculation after expansion of the lower - * half of the vector. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_h vector and the in_l vector are multiplied after - * the lower half of the two-fold sign extension (signed - * halfword to signed word), then stored to the out vector. - * Example : out = __lasx_xvmulwl_w_h(in_h, in_l) - * in_h : 3,-1,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 - * in_l : 2,-1,1,2, 1,0,0, 0, 0,0,1, 0, 1,0,0,1 - * out : 6,1,3,0, 0,0,1,0 - * ============================================================================= - */ -static inline __m256i __lasx_xvmulwl_w_h(__m256i in_h, __m256i in_l) { - __m256i tmp0, tmp1, out; - - tmp0 = __lasx_xvsllwil_w_h(in_h, 0); - tmp1 = __lasx_xvsllwil_w_h(in_l, 0); - out = __lasx_xvmul_w(tmp0, tmp1); - return out; -} - -/* - * ============================================================================= - * Description : Multiplication calculation after expansion of the lower - * half of the vector. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_h vector and the in_l vector are multiplied after - * the lower half of the two-fold sign extension (signed - * halfword to signed word), then stored to the out vector. - * Example : out = __lasx_xvmulwh_w_h(in_h, in_l) - * in_h : 3,-1,3,0, 0,0,0,-1, 0,0,1,-1, 0,0,0,1 - * in_l : 2,-1,1,2, 1,0,0, 0, 0,0,1, 0, 1,0,0,1 - * out : 0,0,0,0, 0,0,0,1 - * ============================================================================= - */ -static inline __m256i __lasx_xvmulwh_w_h(__m256i in_h, __m256i in_l) { - __m256i tmp0, tmp1, out; - - tmp0 = __lasx_xvilvh_h(in_h, in_h); - tmp1 = __lasx_xvilvh_h(in_l, in_l); - out = __lasx_xvmulwev_w_h(tmp0, tmp1); - return out; -} - -/* - * ============================================================================= - * Description : The low half of the vector elements are added to the high half - * after being doubled, then saturated. - * Arguments : Inputs - in_h, in_l - * Output - out - * Details : The in_h vector adds the in_l vector after the lower half of - * the two-fold zero extension (unsigned byte to unsigned - * halfword) and then saturated. The results are stored to the out - * vector. - * Example : out = __lasx_xvsaddw_hu_hu_bu(in_h, in_l) - * in_h : 2,65532,1,2, 1,0,0,0, 0,0,1,0, 1,0,0,1 - * in_l : 3,6,3,0, 0,0,0,1, 0,0,1,1, 0,0,0,1, 3,18,3,0, 0,0,0,1, 0,0,1,1, - * 0,0,0,1 - * out : 5,65535,4,2, 1,0,0,1, 3,18,4,0, 1,0,0,2, - * ============================================================================= - */ -static inline __m256i __lasx_xvsaddw_hu_hu_bu(__m256i in_h, __m256i in_l) { - __m256i tmp1, out; - __m256i zero = {0}; - - tmp1 = __lasx_xvilvl_b(zero, in_l); - out = __lasx_xvsadd_hu(in_h, tmp1); - return out; -} - -/* - * ============================================================================= - * Description : Clip all halfword elements of input vector between min & max - * out = ((in) < (min)) ? (min) : (((in) > (max)) ? (max) : (in)) - * Arguments : Inputs - in (input vector) - * - min (min threshold) - * - max (max threshold) - * Outputs - in (output vector with clipped elements) - * Return Type - signed halfword - * Example : out = __lasx_xvclip_h(in, min, max) - * in : -8,2,280,249, -8,255,280,249, 4,4,4,4, 5,5,5,5 - * min : 1,1,1,1, 1,1,1,1, 1,1,1,1, 1,1,1,1 - * max : 9,9,9,9, 9,9,9,9, 9,9,9,9, 9,9,9,9 - * out : 1,2,9,9, 1,9,9,9, 4,4,4,4, 5,5,5,5 - * ============================================================================= - */ -static inline __m256i __lasx_xvclip_h(__m256i in, __m256i min, __m256i max) { - __m256i out; - - out = __lasx_xvmax_h(min, in); - out = __lasx_xvmin_h(max, out); - return out; -} - -/* - * ============================================================================= - * Description : Clip all signed halfword elements of input vector - * between 0 & 255 - * Arguments : Inputs - in (input vector) - * Outputs - out (output vector with clipped elements) - * Return Type - signed halfword - * Example : See out = __lasx_xvclip255_w(in) - * ============================================================================= - */ -static inline __m256i __lasx_xvclip255_h(__m256i in) { - __m256i out; - - out = __lasx_xvmaxi_h(in, 0); - out = __lasx_xvsat_hu(out, 7); - return out; -} - -/* - * ============================================================================= - * Description : Clip all signed word elements of input vector - * between 0 & 255 - * Arguments : Inputs - in (input vector) - * Output - out (output vector with clipped elements) - * Return Type - signed word - * Example : out = __lasx_xvclip255_w(in) - * in : -8,255,280,249, -8,255,280,249 - * out : 0,255,255,249, 0,255,255,249 - * ============================================================================= - */ -static inline __m256i __lasx_xvclip255_w(__m256i in) { - __m256i out; - - out = __lasx_xvmaxi_w(in, 0); - out = __lasx_xvsat_wu(out, 7); - return out; -} - -/* - * ============================================================================= - * Description : Indexed halfword element values are replicated to all - * elements in output vector. If 'idx < 8' use xvsplati_l_*, - * if 'idx >= 8' use xvsplati_h_*. - * Arguments : Inputs - in, idx - * Output - out - * Details : Idx element value from in vector is replicated to all - * elements in out vector. - * Valid index range for halfword operation is 0-7 - * Example : out = __lasx_xvsplati_l_h(in, idx) - * in : 20,10,11,12, 13,14,15,16, 0,0,2,0, 0,0,0,0 - * idx : 0x02 - * out : 11,11,11,11, 11,11,11,11, 11,11,11,11, 11,11,11,11 - * ============================================================================= - */ -static inline __m256i __lasx_xvsplati_l_h(__m256i in, int idx) { - __m256i out; - - out = __lasx_xvpermi_q(in, in, 0x02); - out = __lasx_xvreplve_h(out, idx); - return out; -} - -/* - * ============================================================================= - * Description : Indexed halfword element values are replicated to all - * elements in output vector. If 'idx < 8' use xvsplati_l_*, - * if 'idx >= 8' use xvsplati_h_*. - * Arguments : Inputs - in, idx - * Output - out - * Details : Idx element value from in vector is replicated to all - * elements in out vector. - * Valid index range for halfword operation is 0-7 - * Example : out = __lasx_xvsplati_h_h(in, idx) - * in : 20,10,11,12, 13,14,15,16, 0,2,0,0, 0,0,0,0 - * idx : 0x09 - * out : 2,2,2,2, 2,2,2,2, 2,2,2,2, 2,2,2,2 - * ============================================================================= - */ -static inline __m256i __lasx_xvsplati_h_h(__m256i in, int idx) { - __m256i out; - - out = __lasx_xvpermi_q(in, in, 0x13); - out = __lasx_xvreplve_h(out, idx); - return out; -} - -/* - * ============================================================================= - * Description : Transpose 4x4 block with double-word elements in vectors - * Arguments : Inputs - _in0, _in1, _in2, _in3 - * Outputs - _out0, _out1, _out2, _out3 - * Example : LASX_TRANSPOSE4x4_D - * _in0 : 1,2,3,4 - * _in1 : 1,2,3,4 - * _in2 : 1,2,3,4 - * _in3 : 1,2,3,4 - * - * _out0 : 1,1,1,1 - * _out1 : 2,2,2,2 - * _out2 : 3,3,3,3 - * _out3 : 4,4,4,4 - * ============================================================================= - */ -#define LASX_TRANSPOSE4x4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, \ - _out3) \ - { \ - __m256i _tmp0, _tmp1, _tmp2, _tmp3; \ - _tmp0 = __lasx_xvilvl_d(_in1, _in0); \ - _tmp1 = __lasx_xvilvh_d(_in1, _in0); \ - _tmp2 = __lasx_xvilvl_d(_in3, _in2); \ - _tmp3 = __lasx_xvilvh_d(_in3, _in2); \ - _out0 = __lasx_xvpermi_q(_tmp2, _tmp0, 0x20); \ - _out2 = __lasx_xvpermi_q(_tmp2, _tmp0, 0x31); \ - _out1 = __lasx_xvpermi_q(_tmp3, _tmp1, 0x20); \ - _out3 = __lasx_xvpermi_q(_tmp3, _tmp1, 0x31); \ - } - -/* - * ============================================================================= - * Description : Transpose 8x8 block with word elements in vectors - * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 - * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, - * _out7 - * Example : LASX_TRANSPOSE8x8_W - * _in0 : 1,2,3,4,5,6,7,8 - * _in1 : 2,2,3,4,5,6,7,8 - * _in2 : 3,2,3,4,5,6,7,8 - * _in3 : 4,2,3,4,5,6,7,8 - * _in4 : 5,2,3,4,5,6,7,8 - * _in5 : 6,2,3,4,5,6,7,8 - * _in6 : 7,2,3,4,5,6,7,8 - * _in7 : 8,2,3,4,5,6,7,8 - * - * _out0 : 1,2,3,4,5,6,7,8 - * _out1 : 2,2,2,2,2,2,2,2 - * _out2 : 3,3,3,3,3,3,3,3 - * _out3 : 4,4,4,4,4,4,4,4 - * _out4 : 5,5,5,5,5,5,5,5 - * _out5 : 6,6,6,6,6,6,6,6 - * _out6 : 7,7,7,7,7,7,7,7 - * _out7 : 8,8,8,8,8,8,8,8 - * ============================================================================= - */ -#define LASX_TRANSPOSE8x8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - __m256i _s0_m, _s1_m; \ - __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ - __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ - \ - _s0_m = __lasx_xvilvl_w(_in2, _in0); \ - _s1_m = __lasx_xvilvl_w(_in3, _in1); \ - _tmp0_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ - _tmp1_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ - _s0_m = __lasx_xvilvh_w(_in2, _in0); \ - _s1_m = __lasx_xvilvh_w(_in3, _in1); \ - _tmp2_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ - _tmp3_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ - _s0_m = __lasx_xvilvl_w(_in6, _in4); \ - _s1_m = __lasx_xvilvl_w(_in7, _in5); \ - _tmp4_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ - _tmp5_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ - _s0_m = __lasx_xvilvh_w(_in6, _in4); \ - _s1_m = __lasx_xvilvh_w(_in7, _in5); \ - _tmp6_m = __lasx_xvilvl_w(_s1_m, _s0_m); \ - _tmp7_m = __lasx_xvilvh_w(_s1_m, _s0_m); \ - _out0 = __lasx_xvpermi_q(_tmp4_m, _tmp0_m, 0x20); \ - _out1 = __lasx_xvpermi_q(_tmp5_m, _tmp1_m, 0x20); \ - _out2 = __lasx_xvpermi_q(_tmp6_m, _tmp2_m, 0x20); \ - _out3 = __lasx_xvpermi_q(_tmp7_m, _tmp3_m, 0x20); \ - _out4 = __lasx_xvpermi_q(_tmp4_m, _tmp0_m, 0x31); \ - _out5 = __lasx_xvpermi_q(_tmp5_m, _tmp1_m, 0x31); \ - _out6 = __lasx_xvpermi_q(_tmp6_m, _tmp2_m, 0x31); \ - _out7 = __lasx_xvpermi_q(_tmp7_m, _tmp3_m, 0x31); \ - } - -/* - * ============================================================================= - * Description : Transpose input 16x8 byte block - * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, - * _in8, _in9, _in10, _in11, _in12, _in13, _in14, _in15 - * (input 16x8 byte block) - * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, - * _out7 (output 8x16 byte block) - * Details : The rows of the matrix become columns, and the columns become - * rows. - * Example : See LASX_TRANSPOSE16x8_H - * ============================================================================= - */ -#define LASX_TRANSPOSE16x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ - _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ - _out6, _out7) \ - { \ - __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ - __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ - \ - _tmp0_m = __lasx_xvilvl_b(_in2, _in0); \ - _tmp1_m = __lasx_xvilvl_b(_in3, _in1); \ - _tmp2_m = __lasx_xvilvl_b(_in6, _in4); \ - _tmp3_m = __lasx_xvilvl_b(_in7, _in5); \ - _tmp4_m = __lasx_xvilvl_b(_in10, _in8); \ - _tmp5_m = __lasx_xvilvl_b(_in11, _in9); \ - _tmp6_m = __lasx_xvilvl_b(_in14, _in12); \ - _tmp7_m = __lasx_xvilvl_b(_in15, _in13); \ - _out0 = __lasx_xvilvl_b(_tmp1_m, _tmp0_m); \ - _out1 = __lasx_xvilvh_b(_tmp1_m, _tmp0_m); \ - _out2 = __lasx_xvilvl_b(_tmp3_m, _tmp2_m); \ - _out3 = __lasx_xvilvh_b(_tmp3_m, _tmp2_m); \ - _out4 = __lasx_xvilvl_b(_tmp5_m, _tmp4_m); \ - _out5 = __lasx_xvilvh_b(_tmp5_m, _tmp4_m); \ - _out6 = __lasx_xvilvl_b(_tmp7_m, _tmp6_m); \ - _out7 = __lasx_xvilvh_b(_tmp7_m, _tmp6_m); \ - _tmp0_m = __lasx_xvilvl_w(_out2, _out0); \ - _tmp2_m = __lasx_xvilvh_w(_out2, _out0); \ - _tmp4_m = __lasx_xvilvl_w(_out3, _out1); \ - _tmp6_m = __lasx_xvilvh_w(_out3, _out1); \ - _tmp1_m = __lasx_xvilvl_w(_out6, _out4); \ - _tmp3_m = __lasx_xvilvh_w(_out6, _out4); \ - _tmp5_m = __lasx_xvilvl_w(_out7, _out5); \ - _tmp7_m = __lasx_xvilvh_w(_out7, _out5); \ - _out0 = __lasx_xvilvl_d(_tmp1_m, _tmp0_m); \ - _out1 = __lasx_xvilvh_d(_tmp1_m, _tmp0_m); \ - _out2 = __lasx_xvilvl_d(_tmp3_m, _tmp2_m); \ - _out3 = __lasx_xvilvh_d(_tmp3_m, _tmp2_m); \ - _out4 = __lasx_xvilvl_d(_tmp5_m, _tmp4_m); \ - _out5 = __lasx_xvilvh_d(_tmp5_m, _tmp4_m); \ - _out6 = __lasx_xvilvl_d(_tmp7_m, _tmp6_m); \ - _out7 = __lasx_xvilvh_d(_tmp7_m, _tmp6_m); \ - } - -/* - * ============================================================================= - * Description : Transpose input 16x8 byte block - * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, - * _in8, _in9, _in10, _in11, _in12, _in13, _in14, _in15 - * (input 16x8 byte block) - * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, - * _out7 (output 8x16 byte block) - * Details : The rows of the matrix become columns, and the columns become - * rows. - * Example : LASX_TRANSPOSE16x8_H - * _in0 : 1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in1 : 2,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in2 : 3,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in3 : 4,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in4 : 5,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in5 : 6,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in6 : 7,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in7 : 8,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in8 : 9,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in9 : 1,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in10 : 0,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in11 : 2,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in12 : 3,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in13 : 7,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in14 : 5,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * _in15 : 6,2,3,4,5,6,7,8,0,0,0,0,0,0,0,0 - * - * _out0 : 1,2,3,4,5,6,7,8,9,1,0,2,3,7,5,6 - * _out1 : 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2 - * _out2 : 3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3 - * _out3 : 4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4 - * _out4 : 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 - * _out5 : 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6 - * _out6 : 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 - * _out7 : 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8 - * ============================================================================= - */ -#define LASX_TRANSPOSE16x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _in8, _in9, _in10, _in11, _in12, _in13, _in14, \ - _in15, _out0, _out1, _out2, _out3, _out4, _out5, \ - _out6, _out7) \ - { \ - __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ - __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ - __m256i _t0, _t1, _t2, _t3, _t4, _t5, _t6, _t7; \ - \ - _tmp0_m = __lasx_xvilvl_h(_in2, _in0); \ - _tmp1_m = __lasx_xvilvl_h(_in3, _in1); \ - _tmp2_m = __lasx_xvilvl_h(_in6, _in4); \ - _tmp3_m = __lasx_xvilvl_h(_in7, _in5); \ - _tmp4_m = __lasx_xvilvl_h(_in10, _in8); \ - _tmp5_m = __lasx_xvilvl_h(_in11, _in9); \ - _tmp6_m = __lasx_xvilvl_h(_in14, _in12); \ - _tmp7_m = __lasx_xvilvl_h(_in15, _in13); \ - _t0 = __lasx_xvilvl_h(_tmp1_m, _tmp0_m); \ - _t1 = __lasx_xvilvh_h(_tmp1_m, _tmp0_m); \ - _t2 = __lasx_xvilvl_h(_tmp3_m, _tmp2_m); \ - _t3 = __lasx_xvilvh_h(_tmp3_m, _tmp2_m); \ - _t4 = __lasx_xvilvl_h(_tmp5_m, _tmp4_m); \ - _t5 = __lasx_xvilvh_h(_tmp5_m, _tmp4_m); \ - _t6 = __lasx_xvilvl_h(_tmp7_m, _tmp6_m); \ - _t7 = __lasx_xvilvh_h(_tmp7_m, _tmp6_m); \ - _tmp0_m = __lasx_xvilvl_d(_t2, _t0); \ - _tmp2_m = __lasx_xvilvh_d(_t2, _t0); \ - _tmp4_m = __lasx_xvilvl_d(_t3, _t1); \ - _tmp6_m = __lasx_xvilvh_d(_t3, _t1); \ - _tmp1_m = __lasx_xvilvl_d(_t6, _t4); \ - _tmp3_m = __lasx_xvilvh_d(_t6, _t4); \ - _tmp5_m = __lasx_xvilvl_d(_t7, _t5); \ - _tmp7_m = __lasx_xvilvh_d(_t7, _t5); \ - _out0 = __lasx_xvpermi_q(_tmp1_m, _tmp0_m, 0x20); \ - _out1 = __lasx_xvpermi_q(_tmp3_m, _tmp2_m, 0x20); \ - _out2 = __lasx_xvpermi_q(_tmp5_m, _tmp4_m, 0x20); \ - _out3 = __lasx_xvpermi_q(_tmp7_m, _tmp6_m, 0x20); \ - \ - _tmp0_m = __lasx_xvilvh_h(_in2, _in0); \ - _tmp1_m = __lasx_xvilvh_h(_in3, _in1); \ - _tmp2_m = __lasx_xvilvh_h(_in6, _in4); \ - _tmp3_m = __lasx_xvilvh_h(_in7, _in5); \ - _tmp4_m = __lasx_xvilvh_h(_in10, _in8); \ - _tmp5_m = __lasx_xvilvh_h(_in11, _in9); \ - _tmp6_m = __lasx_xvilvh_h(_in14, _in12); \ - _tmp7_m = __lasx_xvilvh_h(_in15, _in13); \ - _t0 = __lasx_xvilvl_h(_tmp1_m, _tmp0_m); \ - _t1 = __lasx_xvilvh_h(_tmp1_m, _tmp0_m); \ - _t2 = __lasx_xvilvl_h(_tmp3_m, _tmp2_m); \ - _t3 = __lasx_xvilvh_h(_tmp3_m, _tmp2_m); \ - _t4 = __lasx_xvilvl_h(_tmp5_m, _tmp4_m); \ - _t5 = __lasx_xvilvh_h(_tmp5_m, _tmp4_m); \ - _t6 = __lasx_xvilvl_h(_tmp7_m, _tmp6_m); \ - _t7 = __lasx_xvilvh_h(_tmp7_m, _tmp6_m); \ - _tmp0_m = __lasx_xvilvl_d(_t2, _t0); \ - _tmp2_m = __lasx_xvilvh_d(_t2, _t0); \ - _tmp4_m = __lasx_xvilvl_d(_t3, _t1); \ - _tmp6_m = __lasx_xvilvh_d(_t3, _t1); \ - _tmp1_m = __lasx_xvilvl_d(_t6, _t4); \ - _tmp3_m = __lasx_xvilvh_d(_t6, _t4); \ - _tmp5_m = __lasx_xvilvl_d(_t7, _t5); \ - _tmp7_m = __lasx_xvilvh_d(_t7, _t5); \ - _out4 = __lasx_xvpermi_q(_tmp1_m, _tmp0_m, 0x20); \ - _out5 = __lasx_xvpermi_q(_tmp3_m, _tmp2_m, 0x20); \ - _out6 = __lasx_xvpermi_q(_tmp5_m, _tmp4_m, 0x20); \ - _out7 = __lasx_xvpermi_q(_tmp7_m, _tmp6_m, 0x20); \ - } - -/* - * ============================================================================= - * Description : Transpose 4x4 block with halfword elements in vectors - * Arguments : Inputs - _in0, _in1, _in2, _in3 - * Outputs - _out0, _out1, _out2, _out3 - * Return Type - signed halfword - * Details : The rows of the matrix become columns, and the columns become - * rows. - * Example : See LASX_TRANSPOSE8x8_H - * ============================================================================= - */ -#define LASX_TRANSPOSE4x4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, \ - _out3) \ - { \ - __m256i _s0_m, _s1_m; \ - \ - _s0_m = __lasx_xvilvl_h(_in1, _in0); \ - _s1_m = __lasx_xvilvl_h(_in3, _in2); \ - _out0 = __lasx_xvilvl_w(_s1_m, _s0_m); \ - _out2 = __lasx_xvilvh_w(_s1_m, _s0_m); \ - _out1 = __lasx_xvilvh_d(_out0, _out0); \ - _out3 = __lasx_xvilvh_d(_out2, _out2); \ - } - -/* - * ============================================================================= - * Description : Transpose input 8x8 byte block - * Arguments : Inputs - _in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7 - * (input 8x8 byte block) - * Outputs - _out0, _out1, _out2, _out3, _out4, _out5, _out6, - * _out7 (output 8x8 byte block) - * Example : See LASX_TRANSPOSE8x8_H - * ============================================================================= - */ -#define LASX_TRANSPOSE8x8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ - __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ - _tmp0_m = __lasx_xvilvl_b(_in2, _in0); \ - _tmp1_m = __lasx_xvilvl_b(_in3, _in1); \ - _tmp2_m = __lasx_xvilvl_b(_in6, _in4); \ - _tmp3_m = __lasx_xvilvl_b(_in7, _in5); \ - _tmp4_m = __lasx_xvilvl_b(_tmp1_m, _tmp0_m); \ - _tmp5_m = __lasx_xvilvh_b(_tmp1_m, _tmp0_m); \ - _tmp6_m = __lasx_xvilvl_b(_tmp3_m, _tmp2_m); \ - _tmp7_m = __lasx_xvilvh_b(_tmp3_m, _tmp2_m); \ - _out0 = __lasx_xvilvl_w(_tmp6_m, _tmp4_m); \ - _out2 = __lasx_xvilvh_w(_tmp6_m, _tmp4_m); \ - _out4 = __lasx_xvilvl_w(_tmp7_m, _tmp5_m); \ - _out6 = __lasx_xvilvh_w(_tmp7_m, _tmp5_m); \ - _out1 = __lasx_xvbsrl_v(_out0, 8); \ - _out3 = __lasx_xvbsrl_v(_out2, 8); \ - _out5 = __lasx_xvbsrl_v(_out4, 8); \ - _out7 = __lasx_xvbsrl_v(_out6, 8); \ - } - -/* - * ============================================================================= - * Description : Transpose 8x8 block with halfword elements in vectors. - * Arguments : Inputs - _in0, _in1, ~ - * Outputs - _out0, _out1, ~ - * Details : The rows of the matrix become columns, and the columns become - * rows. - * Example : LASX_TRANSPOSE8x8_H - * _in0 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * _in1 : 8,2,3,4, 5,6,7,8, 8,2,3,4, 5,6,7,8 - * _in2 : 8,2,3,4, 5,6,7,8, 8,2,3,4, 5,6,7,8 - * _in3 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * _in4 : 9,2,3,4, 5,6,7,8, 9,2,3,4, 5,6,7,8 - * _in5 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * _in6 : 1,2,3,4, 5,6,7,8, 1,2,3,4, 5,6,7,8 - * _in7 : 9,2,3,4, 5,6,7,8, 9,2,3,4, 5,6,7,8 - * - * _out0 : 1,8,8,1, 9,1,1,9, 1,8,8,1, 9,1,1,9 - * _out1 : 2,2,2,2, 2,2,2,2, 2,2,2,2, 2,2,2,2 - * _out2 : 3,3,3,3, 3,3,3,3, 3,3,3,3, 3,3,3,3 - * _out3 : 4,4,4,4, 4,4,4,4, 4,4,4,4, 4,4,4,4 - * _out4 : 5,5,5,5, 5,5,5,5, 5,5,5,5, 5,5,5,5 - * _out5 : 6,6,6,6, 6,6,6,6, 6,6,6,6, 6,6,6,6 - * _out6 : 7,7,7,7, 7,7,7,7, 7,7,7,7, 7,7,7,7 - * _out7 : 8,8,8,8, 8,8,8,8, 8,8,8,8, 8,8,8,8 - * ============================================================================= - */ -#define LASX_TRANSPOSE8x8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - __m256i _s0_m, _s1_m; \ - __m256i _tmp0_m, _tmp1_m, _tmp2_m, _tmp3_m; \ - __m256i _tmp4_m, _tmp5_m, _tmp6_m, _tmp7_m; \ - \ - _s0_m = __lasx_xvilvl_h(_in6, _in4); \ - _s1_m = __lasx_xvilvl_h(_in7, _in5); \ - _tmp0_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ - _tmp1_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ - _s0_m = __lasx_xvilvh_h(_in6, _in4); \ - _s1_m = __lasx_xvilvh_h(_in7, _in5); \ - _tmp2_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ - _tmp3_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ - \ - _s0_m = __lasx_xvilvl_h(_in2, _in0); \ - _s1_m = __lasx_xvilvl_h(_in3, _in1); \ - _tmp4_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ - _tmp5_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ - _s0_m = __lasx_xvilvh_h(_in2, _in0); \ - _s1_m = __lasx_xvilvh_h(_in3, _in1); \ - _tmp6_m = __lasx_xvilvl_h(_s1_m, _s0_m); \ - _tmp7_m = __lasx_xvilvh_h(_s1_m, _s0_m); \ - \ - _out0 = __lasx_xvpickev_d(_tmp0_m, _tmp4_m); \ - _out2 = __lasx_xvpickev_d(_tmp1_m, _tmp5_m); \ - _out4 = __lasx_xvpickev_d(_tmp2_m, _tmp6_m); \ - _out6 = __lasx_xvpickev_d(_tmp3_m, _tmp7_m); \ - _out1 = __lasx_xvpickod_d(_tmp0_m, _tmp4_m); \ - _out3 = __lasx_xvpickod_d(_tmp1_m, _tmp5_m); \ - _out5 = __lasx_xvpickod_d(_tmp2_m, _tmp6_m); \ - _out7 = __lasx_xvpickod_d(_tmp3_m, _tmp7_m); \ - } - -/* - * ============================================================================= - * Description : Butterfly of 4 input vectors - * Arguments : Inputs - _in0, _in1, _in2, _in3 - * Outputs - _out0, _out1, _out2, _out3 - * Details : Butterfly operation - * Example : LASX_BUTTERFLY_4 - * _out0 = _in0 + _in3; - * _out1 = _in1 + _in2; - * _out2 = _in1 - _in2; - * _out3 = _in0 - _in3; - * ============================================================================= - */ -#define LASX_BUTTERFLY_4_B(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - _out0 = __lasx_xvadd_b(_in0, _in3); \ - _out1 = __lasx_xvadd_b(_in1, _in2); \ - _out2 = __lasx_xvsub_b(_in1, _in2); \ - _out3 = __lasx_xvsub_b(_in0, _in3); \ - } -#define LASX_BUTTERFLY_4_H(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - _out0 = __lasx_xvadd_h(_in0, _in3); \ - _out1 = __lasx_xvadd_h(_in1, _in2); \ - _out2 = __lasx_xvsub_h(_in1, _in2); \ - _out3 = __lasx_xvsub_h(_in0, _in3); \ - } -#define LASX_BUTTERFLY_4_W(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - _out0 = __lasx_xvadd_w(_in0, _in3); \ - _out1 = __lasx_xvadd_w(_in1, _in2); \ - _out2 = __lasx_xvsub_w(_in1, _in2); \ - _out3 = __lasx_xvsub_w(_in0, _in3); \ - } -#define LASX_BUTTERFLY_4_D(_in0, _in1, _in2, _in3, _out0, _out1, _out2, _out3) \ - { \ - _out0 = __lasx_xvadd_d(_in0, _in3); \ - _out1 = __lasx_xvadd_d(_in1, _in2); \ - _out2 = __lasx_xvsub_d(_in1, _in2); \ - _out3 = __lasx_xvsub_d(_in0, _in3); \ - } - -/* - * ============================================================================= - * Description : Butterfly of 8 input vectors - * Arguments : Inputs - _in0, _in1, _in2, _in3, ~ - * Outputs - _out0, _out1, _out2, _out3, ~ - * Details : Butterfly operation - * Example : LASX_BUTTERFLY_8 - * _out0 = _in0 + _in7; - * _out1 = _in1 + _in6; - * _out2 = _in2 + _in5; - * _out3 = _in3 + _in4; - * _out4 = _in3 - _in4; - * _out5 = _in2 - _in5; - * _out6 = _in1 - _in6; - * _out7 = _in0 - _in7; - * ============================================================================= - */ -#define LASX_BUTTERFLY_8_B(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - _out0 = __lasx_xvadd_b(_in0, _in7); \ - _out1 = __lasx_xvadd_b(_in1, _in6); \ - _out2 = __lasx_xvadd_b(_in2, _in5); \ - _out3 = __lasx_xvadd_b(_in3, _in4); \ - _out4 = __lasx_xvsub_b(_in3, _in4); \ - _out5 = __lasx_xvsub_b(_in2, _in5); \ - _out6 = __lasx_xvsub_b(_in1, _in6); \ - _out7 = __lasx_xvsub_b(_in0, _in7); \ - } - -#define LASX_BUTTERFLY_8_H(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - _out0 = __lasx_xvadd_h(_in0, _in7); \ - _out1 = __lasx_xvadd_h(_in1, _in6); \ - _out2 = __lasx_xvadd_h(_in2, _in5); \ - _out3 = __lasx_xvadd_h(_in3, _in4); \ - _out4 = __lasx_xvsub_h(_in3, _in4); \ - _out5 = __lasx_xvsub_h(_in2, _in5); \ - _out6 = __lasx_xvsub_h(_in1, _in6); \ - _out7 = __lasx_xvsub_h(_in0, _in7); \ - } - -#define LASX_BUTTERFLY_8_W(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - _out0 = __lasx_xvadd_w(_in0, _in7); \ - _out1 = __lasx_xvadd_w(_in1, _in6); \ - _out2 = __lasx_xvadd_w(_in2, _in5); \ - _out3 = __lasx_xvadd_w(_in3, _in4); \ - _out4 = __lasx_xvsub_w(_in3, _in4); \ - _out5 = __lasx_xvsub_w(_in2, _in5); \ - _out6 = __lasx_xvsub_w(_in1, _in6); \ - _out7 = __lasx_xvsub_w(_in0, _in7); \ - } - -#define LASX_BUTTERFLY_8_D(_in0, _in1, _in2, _in3, _in4, _in5, _in6, _in7, \ - _out0, _out1, _out2, _out3, _out4, _out5, _out6, \ - _out7) \ - { \ - _out0 = __lasx_xvadd_d(_in0, _in7); \ - _out1 = __lasx_xvadd_d(_in1, _in6); \ - _out2 = __lasx_xvadd_d(_in2, _in5); \ - _out3 = __lasx_xvadd_d(_in3, _in4); \ - _out4 = __lasx_xvsub_d(_in3, _in4); \ - _out5 = __lasx_xvsub_d(_in2, _in5); \ - _out6 = __lasx_xvsub_d(_in1, _in6); \ - _out7 = __lasx_xvsub_d(_in0, _in7); \ - } - -#endif // LASX - -/* - * ============================================================================= - * Description : Print out elements in vector. - * Arguments : Inputs - RTYPE, _element_num, _in0, _enter - * Outputs - - * Details : Print out '_element_num' elements in 'RTYPE' vector '_in0', if - * '_enter' is TRUE, prefix "\nVP:" will be added first. - * Example : VECT_PRINT(v4i32,4,in0,1); // in0: 1,2,3,4 - * VP:1,2,3,4, - * ============================================================================= - */ -#define VECT_PRINT(RTYPE, element_num, in0, enter) \ - { \ - RTYPE _tmp0 = (RTYPE)in0; \ - int _i = 0; \ - if (enter) \ - printf("\nVP:"); \ - for (_i = 0; _i < element_num; _i++) \ - printf("%d,", _tmp0[_i]); \ - } - -#endif /* LOONGSON_INTRINSICS_H */ -#endif /* INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H */ +#endif // INCLUDE_LIBYUV_LOONGSON_INTRINSICS_H_ diff --git a/include/libyuv/rotate_row.h b/include/libyuv/rotate_row.h index d73a2f72b..47a911753 100644 --- a/include/libyuv/rotate_row.h +++ b/include/libyuv/rotate_row.h @@ -93,13 +93,15 @@ extern "C" { #endif #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) -#define HAS_TRANSPOSEWX16_LSX -#define HAS_TRANSPOSEUVWX16_LSX +#define HAS_TRANSPOSEWX8_LSX +#define HAS_TRANSPOSEUVWX8_LSX #define HAS_TRANSPOSEWX8_16_LSX #define HAS_TRANSPOSEWX4_32_LSX #endif #if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) +#define HAS_TRANSPOSEWX16_LASX +#define HAS_TRANSPOSEUVWX16_LASX #define HAS_TRANSPOSEWX16_16_LASX #define HAS_TRANSPOSEWX4_32_LASX #endif @@ -167,11 +169,16 @@ void TransposeWx16_AVX512BW(const uint8_t* src, uint8_t* dst, int dst_stride, int width); -void TransposeWx16_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); +void TransposeWx8_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); +void TransposeWx16_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx8_WASMSIMD(const uint8_t* src, int src_stride, uint8_t* dst, @@ -203,11 +210,16 @@ void TransposeWx16_Any_AVX2(const uint8_t* src, uint8_t* dst, int dst_stride, int width); -void TransposeWx16_Any_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width); +void TransposeWx8_Any_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); +void TransposeWx16_Any_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width); void TransposeWx8_Any_WASMSIMD(const uint8_t* src, int src_stride, uint8_t* dst, @@ -278,13 +290,20 @@ void TransposeUVWxH_SME(const uint8_t* src, int dst_stride_b, int width, int height); -void TransposeUVWx16_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width); +void TransposeUVWx8_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); +void TransposeUVWx16_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeUVWx8_WASMSIMD(const uint8_t* src, int src_stride, uint8_t* dst_a, @@ -321,13 +340,20 @@ void TransposeUVWx16_Any_AVX2(const uint8_t* src, uint8_t* dst_b, int dst_stride_b, int width); -void TransposeUVWx16_Any_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width); +void TransposeUVWx8_Any_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); +void TransposeUVWx16_Any_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width); void TransposeUVWx8_Any_WASMSIMD(const uint8_t* src, int src_stride, uint8_t* dst_a, diff --git a/include/libyuv/row.h b/include/libyuv/row.h index bf01f1c59..73ec309fd 100644 --- a/include/libyuv/row.h +++ b/include/libyuv/row.h @@ -837,10 +837,15 @@ extern "C" { #define HAS_ARGB4444TOARGBROW_LASX #define HAS_ARGBADDROW_LASX #define HAS_ARGBATTENUATEROW_LASX +#define HAS_ARGBBLENDROW_LASX +#define HAS_ARGBCOLORMATRIXROW_LASX +#define HAS_ARGBEXTRACTALPHAROW_LASX #define HAS_ARGBGRAYROW_LASX #define HAS_ARGBMIRRORROW_LASX #define HAS_ARGBMULTIPLYROW_LASX +#define HAS_ARGBQUANTIZEROW_LASX #define HAS_ARGBSEPIAROW_LASX +#define HAS_ARGBSETROW_LASX #define HAS_ARGBSHADEROW_LASX #define HAS_ARGBSHUFFLEROW_LASX #define HAS_ARGBSUBTRACTROW_LASX @@ -852,6 +857,7 @@ extern "C" { #define HAS_ARGBTORGB565ROW_LASX #define HAS_ARGBTOYMATRIXROW_LASX #define HAS_COPYROW_LASX +#define HAS_I400TOARGBROW_LASX #define HAS_I422ALPHATOARGBROW_LASX #define HAS_I422TOARGB1555ROW_LASX #define HAS_I422TOARGB4444ROW_LASX @@ -861,7 +867,12 @@ extern "C" { #define HAS_I422TORGBAROW_LASX #define HAS_I422TOUYVYROW_LASX #define HAS_I422TOYUY2ROW_LASX +#define HAS_I444TOARGBROW_LASX +#define HAS_INTERPOLATEROW_LASX +#define HAS_J400TOARGBROW_LASX +#define HAS_MERGEUVROW_LASX #define HAS_MIRRORROW_LASX +#define HAS_MIRRORSPLITUVROW_LASX #define HAS_MIRRORUVROW_LASX #define HAS_NV12TOARGBROW_LASX #define HAS_NV12TORGB565ROW_LASX @@ -871,13 +882,20 @@ extern "C" { #define HAS_RAWTOUVROW_LASX #define HAS_RGB24TOARGBROW_LASX #define HAS_RGB24TOUVROW_LASX -#define HAS_RGBTOYMATRIXROW_LASX #define HAS_RGB565TOARGBROW_LASX #define HAS_RGB565TOUVROW_LASX #define HAS_RGB565TOYROW_LASX +#define HAS_RGBTOYMATRIXROW_LASX +#define HAS_SETROW_LASX +#define HAS_SOBELROW_LASX +#define HAS_SOBELTOPLANEROW_LASX +#define HAS_SOBELXYROW_LASX +#define HAS_SPLITUVROW_LASX +#define HAS_UYVYTOARGBROW_LASX #define HAS_UYVYTOUV422ROW_LASX #define HAS_UYVYTOUVROW_LASX #define HAS_UYVYTOYROW_LASX +#define HAS_YUY2TOARGBROW_LASX #define HAS_YUY2TOUV422ROW_LASX #define HAS_YUY2TOUVROW_LASX #define HAS_YUY2TOYROW_LASX @@ -1675,6 +1693,12 @@ void I444ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I444ToARGBRow_LASX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGBRow_LSX(const uint8_t* src_y, const uint8_t* src_u, @@ -1797,10 +1821,18 @@ void YUY2ToARGBRow_LSX(const uint8_t* src_yuy2, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void YUY2ToARGBRow_LASX(const uint8_t* src_yuy2, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void UYVYToARGBRow_LSX(const uint8_t* src_uyvy, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void UYVYToARGBRow_LASX(const uint8_t* src_uyvy, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void NV12ToARGBRow_RVV(const uint8_t* src_y, const uint8_t* src_uv, uint8_t* dst_argb, @@ -2615,16 +2647,31 @@ void ABGRToUVRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, int width); +void ABGRToUVRow_Any_LASX(const uint8_t* src_ptr, + int src_stride_ptr, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void BGRAToUVRow_Any_LSX(const uint8_t* src_ptr, int src_stride_ptr, uint8_t* dst_u, uint8_t* dst_v, int width); +void BGRAToUVRow_Any_LASX(const uint8_t* src_ptr, + int src_stride_ptr, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void RGBAToUVRow_Any_LSX(const uint8_t* src_ptr, int src_stride_ptr, uint8_t* dst_u, uint8_t* dst_v, int width); +void RGBAToUVRow_Any_LASX(const uint8_t* src_ptr, + int src_stride_ptr, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void ARGB1555ToUVRow_Any_LSX(const uint8_t* src_ptr, int src_stride_ptr, uint8_t* dst_u, @@ -2775,6 +2822,10 @@ void MirrorSplitUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width); +void MirrorSplitUVRow_LASX(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void MirrorSplitUVRow_C(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, @@ -2850,6 +2901,10 @@ void SplitUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, int width); +void SplitUVRow_LASX(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void SplitUVRow_RVV(const uint8_t* src_uv, uint8_t* dst_u, uint8_t* dst_v, @@ -2874,6 +2929,10 @@ void SplitUVRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_u, uint8_t* dst_v, int width); +void SplitUVRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_u, + uint8_t* dst_v, + int width); void DetileRow_C(const uint8_t* src, ptrdiff_t src_tile_stride, uint8_t* dst, @@ -3019,6 +3078,10 @@ void MergeUVRow_LSX(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, int width); +void MergeUVRow_LASX(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width); void MergeUVRow_RVV(const uint8_t* src_u, const uint8_t* src_v, uint8_t* dst_uv, @@ -3043,6 +3106,10 @@ void MergeUVRow_Any_LSX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); +void MergeUVRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); void HalfMergeUVRow_C(const uint8_t* src_u, int src_stride_u, @@ -3953,6 +4020,9 @@ void ARGBExtractAlphaRow_NEON(const uint8_t* src_argb, void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb, uint8_t* dst_a, int width); +void ARGBExtractAlphaRow_LASX(const uint8_t* src_argb, + uint8_t* dst_a, + int width); void ARGBExtractAlphaRow_WASMSIMD(const uint8_t* src_argb, uint8_t* dst_a, int width); @@ -3971,6 +4041,9 @@ void ARGBExtractAlphaRow_Any_NEON(const uint8_t* src_ptr, void ARGBExtractAlphaRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void ARGBExtractAlphaRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void ARGBExtractAlphaRow_Any_WASMSIMD(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); @@ -4000,16 +4073,20 @@ void SetRow_X86(uint8_t* dst, uint8_t v8, int width); void SetRow_ERMS(uint8_t* dst, uint8_t v8, int width); void SetRow_NEON(uint8_t* dst, uint8_t v8, int width); void SetRow_LSX(uint8_t* dst, uint8_t v8, int width); +void SetRow_LASX(uint8_t* dst, uint8_t v8, int width); void SetRow_Any_X86(uint8_t* dst_ptr, uint8_t v32, int width); void SetRow_Any_NEON(uint8_t* dst_ptr, uint8_t v32, int width); void SetRow_Any_LSX(uint8_t* dst_ptr, uint8_t v32, int width); +void SetRow_Any_LASX(uint8_t* dst_ptr, uint8_t v32, int width); void ARGBSetRow_C(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_X86(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_NEON(uint8_t* dst, uint32_t v32, int width); void ARGBSetRow_Any_NEON(uint8_t* dst_ptr, uint32_t v32, int width); void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width); +void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width); void ARGBSetRow_Any_LSX(uint8_t* dst_ptr, uint32_t v32, int width); +void ARGBSetRow_Any_LASX(uint8_t* dst_ptr, uint32_t v32, int width); // ARGBShufflers for BGRAToARGB etc. void ARGBShuffleRow_C(const uint8_t* src_argb, @@ -4438,6 +4515,7 @@ void J400ToARGBRow_AVX512BW(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_AVX2(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_NEON(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, int width); +void J400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_C(const uint8_t* src_y, uint8_t* dst_argb, int width); void J400ToARGBRow_Any_AVX512BW(const uint8_t* src_ptr, @@ -4450,6 +4528,9 @@ void J400ToARGBRow_Any_NEON(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); void J400ToARGBRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, int width); +void J400ToARGBRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + int width); void I444ToARGBRow_C(const uint8_t* src_y, const uint8_t* src_u, @@ -5400,6 +5481,10 @@ void I400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width); +void I400ToARGBRow_LASX(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width); void I400ToARGBRow_RVV(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, @@ -5420,6 +5505,10 @@ void I400ToARGBRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, int width); +void I400ToARGBRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); // ARGB preattenuated alpha blend. void ARGBBlendRow_SSSE3(const uint8_t* src_argb, @@ -5434,10 +5523,18 @@ void ARGBBlendRow_LSX(const uint8_t* src_argb0, const uint8_t* src_argb1, uint8_t* dst_argb, int width); +void ARGBBlendRow_LASX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); void ARGBBlendRow_Any_LSX(const uint8_t* src_argb0, const uint8_t* src_argb1, uint8_t* dst_argb, int width); +void ARGBBlendRow_Any_LASX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width); void ARGBBlendRow_RVV(const uint8_t* src_argb0, const uint8_t* src_argb1, uint8_t* dst_argb, @@ -5982,6 +6079,12 @@ void I444ToARGBRow_Any_LSX(const uint8_t* y_buf, uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, int width); +void I444ToARGBRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* u_buf, + const uint8_t* v_buf, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); void I422ToARGBRow_Any_LSX(const uint8_t* y_buf, const uint8_t* u_buf, const uint8_t* v_buf, @@ -6103,10 +6206,18 @@ void YUY2ToARGBRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, int width); +void YUY2ToARGBRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); void UYVYToARGBRow_Any_LSX(const uint8_t* src_ptr, uint8_t* dst_ptr, const struct YuvConstants* yuvconstants, int width); +void UYVYToARGBRow_Any_LASX(const uint8_t* src_ptr, + uint8_t* dst_ptr, + const struct YuvConstants* yuvconstants, + int width); void YUY2ToYRow_AVX2(const uint8_t* src_yuy2, uint8_t* dst_y, int width); void YUY2ToUVRow_AVX2(const uint8_t* src_yuy2, @@ -6632,6 +6743,10 @@ void ARGBColorMatrixRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, const int8_t* matrix_argb, int width); +void ARGBColorMatrixRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width); void ARGBColorTableRow_C(uint8_t* dst_argb, const uint8_t* table_argb, @@ -6667,6 +6782,11 @@ void ARGBQuantizeRow_LSX(uint8_t* dst_argb, int interval_size, int interval_offset, int width); +void ARGBQuantizeRow_LASX(uint8_t* dst_argb, + int scale, + int interval_size, + int interval_offset, + int width); void ARGBShadeRow_C(const uint8_t* src_argb, uint8_t* dst_argb, @@ -6781,6 +6901,11 @@ void InterpolateRow_LSX(uint8_t* dst_ptr, ptrdiff_t src_stride, int width, int source_y_fraction); +void InterpolateRow_LASX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int source_y_fraction); void InterpolateRow_RVV(uint8_t* dst_ptr, const uint8_t* src_ptr, ptrdiff_t src_stride, @@ -6801,6 +6926,11 @@ void InterpolateRow_Any_LSX(uint8_t* dst_ptr, ptrdiff_t src_stride_ptr, int width, int source_y_fraction); +void InterpolateRow_Any_LASX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride_ptr, + int width, + int source_y_fraction); void InterpolateRow_16_C(uint16_t* dst_ptr, const uint16_t* src_ptr, @@ -6877,6 +7007,10 @@ void SobelRow_LSX(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); +void SobelRow_LASX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width); void SobelToPlaneRow_C(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, @@ -6893,6 +7027,10 @@ void SobelToPlaneRow_LSX(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_y, int width); +void SobelToPlaneRow_LASX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width); void SobelXYRow_C(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, @@ -6909,6 +7047,10 @@ void SobelXYRow_LSX(const uint8_t* src_sobelx, const uint8_t* src_sobely, uint8_t* dst_argb, int width); +void SobelXYRow_LASX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width); void SobelRow_Any_SSE2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, @@ -6921,6 +7063,10 @@ void SobelRow_Any_LSX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); +void SobelRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); void SobelToPlaneRow_Any_SSE2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, @@ -6933,6 +7079,10 @@ void SobelToPlaneRow_Any_LSX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); +void SobelToPlaneRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); void SobelXYRow_Any_SSE2(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, @@ -6945,6 +7095,10 @@ void SobelXYRow_Any_LSX(const uint8_t* y_buf, const uint8_t* uv_buf, uint8_t* dst_ptr, int width); +void SobelXYRow_Any_LASX(const uint8_t* y_buf, + const uint8_t* uv_buf, + uint8_t* dst_ptr, + int width); void ARGBPolynomialRow_C(const uint8_t* src_argb, uint8_t* dst_argb, @@ -7005,10 +7159,18 @@ void HalfFloatRow_LSX(const uint16_t* src, uint16_t* dst, float scale, int width); +void HalfFloatRow_LASX(const uint16_t* src, + uint16_t* dst, + float scale, + int width); void HalfFloatRow_Any_LSX(const uint16_t* src_ptr, uint16_t* dst_ptr, float param, int width); +void HalfFloatRow_Any_LASX(const uint16_t* src_ptr, + uint16_t* dst_ptr, + float param, + int width); void ByteToFloatRow_C(const uint8_t* src, float* dst, float scale, int width); void ByteToFloatRow_NEON(const uint8_t* src, float* dst, diff --git a/source/convert.cc b/source/convert.cc index 075b3e869..74e236808 100644 --- a/source/convert.cc +++ b/source/convert.cc @@ -997,6 +997,14 @@ int I010ToNV12(const uint16_t* src_y, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -1496,6 +1504,14 @@ int I422ToNV21(const uint8_t* src_y, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -1543,11 +1559,19 @@ int I422ToNV21(const uint8_t* src_y, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { InterpolateRow = InterpolateRow_LSX; } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; @@ -2085,11 +2109,19 @@ static int Px10ToI420(const uint16_t* src_y, #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; - if (IS_ALIGNED(src_uv_width, 32)) { + if (IS_ALIGNED(src_uv_width, 16)) { SplitUVRow = SplitUVRow_LSX; } } #endif +#if defined(HAS_SPLITUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitUVRow = SplitUVRow_Any_LASX; + if (IS_ALIGNED(src_uv_width, 32)) { + SplitUVRow = SplitUVRow_LASX; + } + } +#endif #if defined(HAS_SPLITUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { SplitUVRow = SplitUVRow_RVV; @@ -2564,6 +2596,16 @@ int UYVYToI420(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + UYVYToYRow = UYVYToYRow_Any_LASX; + UYVYToUVRow = UYVYToUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + UYVYToYRow = UYVYToYRow_LASX; + UYVYToUVRow = UYVYToUVRow_LASX; + } + } +#endif #if defined(HAS_UYVYTOYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { UYVYToYRow = UYVYToYRow_Any_LSX; @@ -2851,7 +2893,7 @@ int ARGBToI420Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -2859,7 +2901,7 @@ int ARGBToI420Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -3083,7 +3125,7 @@ int ARGBToI420Alpha(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -3091,7 +3133,7 @@ int ARGBToI420Alpha(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -3200,8 +3242,14 @@ int ARGBToI420Alpha(const uint8_t* src_argb, #endif #if defined(HAS_ARGBEXTRACTALPHAROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { - ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_LSX - : ARGBExtractAlphaRow_Any_LSX; + ARGBExtractAlphaRow = IS_ALIGNED(width, 8) ? ARGBExtractAlphaRow_LSX + : ARGBExtractAlphaRow_Any_LSX; + } +#endif +#if defined(HAS_ARGBEXTRACTALPHAROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_LASX + : ARGBExtractAlphaRow_Any_LASX; } #endif #if defined(HAS_ARGBEXTRACTALPHAROW_WASMSIMD) diff --git a/source/convert_argb.cc b/source/convert_argb.cc index a889889c0..80852a463 100644 --- a/source/convert_argb.cc +++ b/source/convert_argb.cc @@ -123,7 +123,7 @@ int I420ToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToARGBRow = I422ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToARGBRow = I422ToARGBRow_LSX; } } @@ -131,7 +131,7 @@ int I420ToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToARGBRow = I422ToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToARGBRow = I422ToARGBRow_LASX; } } @@ -386,7 +386,7 @@ int I422ToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToARGBRow = I422ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToARGBRow = I422ToARGBRow_LSX; } } @@ -394,7 +394,7 @@ int I422ToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToARGBRow = I422ToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToARGBRow = I422ToARGBRow_LASX; } } @@ -640,11 +640,19 @@ int I444ToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I444TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I444ToARGBRow = I444ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I444ToARGBRow = I444ToARGBRow_LSX; } } #endif +#if defined(HAS_I444TOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + I444ToARGBRow = I444ToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + I444ToARGBRow = I444ToARGBRow_LASX; + } + } +#endif #if defined(HAS_I444TOARGBROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I444ToARGBRow = I444ToARGBRow_RVV; @@ -2450,7 +2458,7 @@ int I420AlphaToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I422ALPHATOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422AlphaToARGBRow = I422AlphaToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422AlphaToARGBRow = I422AlphaToARGBRow_LSX; } } @@ -2500,7 +2508,7 @@ int I420AlphaToARGBMatrix(const uint8_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -2508,7 +2516,7 @@ int I420AlphaToARGBMatrix(const uint8_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -2612,7 +2620,7 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I422ALPHATOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422AlphaToARGBRow = I422AlphaToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422AlphaToARGBRow = I422AlphaToARGBRow_LSX; } } @@ -2662,7 +2670,7 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -2670,7 +2678,7 @@ int I422AlphaToARGBMatrix(const uint8_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -2806,7 +2814,7 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -2814,7 +2822,7 @@ int I444AlphaToARGBMatrix(const uint8_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -3065,7 +3073,7 @@ int I010AlphaToARGBMatrix(const uint16_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -3073,7 +3081,7 @@ int I010AlphaToARGBMatrix(const uint16_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -3197,7 +3205,7 @@ int I210AlphaToARGBMatrix(const uint16_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -3205,7 +3213,7 @@ int I210AlphaToARGBMatrix(const uint16_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -3327,7 +3335,7 @@ int I410AlphaToARGBMatrix(const uint16_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -3335,7 +3343,7 @@ int I410AlphaToARGBMatrix(const uint16_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -3423,11 +3431,19 @@ int I400ToARGBMatrix(const uint8_t* src_y, #if defined(HAS_I400TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I400ToARGBRow = I400ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I400ToARGBRow = I400ToARGBRow_LSX; } } #endif +#if defined(HAS_I400TOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + I400ToARGBRow = I400ToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + I400ToARGBRow = I400ToARGBRow_LASX; + } + } +#endif #if defined(HAS_I400TOARGBROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { I400ToARGBRow = I400ToARGBRow_RVV; @@ -3509,11 +3525,19 @@ int J400ToARGB(const uint8_t* src_y, #if defined(HAS_J400TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { J400ToARGBRow = J400ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { J400ToARGBRow = J400ToARGBRow_LSX; } } #endif +#if defined(HAS_J400TOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + J400ToARGBRow = J400ToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + J400ToARGBRow = J400ToARGBRow_LASX; + } + } +#endif #if defined(HAS_J400TOARGBROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { J400ToARGBRow = J400ToARGBRow_RVV; @@ -4130,7 +4154,7 @@ int RGB565ToARGB(const uint8_t* src_rgb565, #if defined(HAS_RGB565TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { RGB565ToARGBRow = RGB565ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { RGB565ToARGBRow = RGB565ToARGBRow_LSX; } } @@ -4138,7 +4162,7 @@ int RGB565ToARGB(const uint8_t* src_rgb565, #if defined(HAS_RGB565TOARGBROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { RGB565ToARGBRow = RGB565ToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { RGB565ToARGBRow = RGB565ToARGBRow_LASX; } } @@ -4213,7 +4237,7 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555, #if defined(HAS_ARGB1555TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGB1555ToARGBRow = ARGB1555ToARGBRow_LSX; } } @@ -4221,7 +4245,7 @@ int ARGB1555ToARGB(const uint8_t* src_argb1555, #if defined(HAS_ARGB1555TOARGBROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGB1555ToARGBRow = ARGB1555ToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGB1555ToARGBRow = ARGB1555ToARGBRow_LASX; } } @@ -4291,7 +4315,7 @@ int ARGB4444ToARGB(const uint8_t* src_argb4444, #if defined(HAS_ARGB4444TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGB4444ToARGBRow = ARGB4444ToARGBRow_LSX; } } @@ -4299,7 +4323,7 @@ int ARGB4444ToARGB(const uint8_t* src_argb4444, #if defined(HAS_ARGB4444TOARGBROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGB4444ToARGBRow = ARGB4444ToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGB4444ToARGBRow = ARGB4444ToARGBRow_LASX; } } @@ -5156,6 +5180,14 @@ int YUY2ToARGBMatrix(const uint8_t* src_yuy2, YUY2ToARGBRow = YUY2ToARGBRow_LSX; } } +#endif +#if defined(HAS_YUY2TOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + YUY2ToARGBRow = YUY2ToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + YUY2ToARGBRow = YUY2ToARGBRow_LASX; + } + } #endif for (y = 0; y < height; ++y) { YUY2ToARGBRow(src_yuy2, dst_argb, yuvconstants, width); @@ -5248,6 +5280,14 @@ int UYVYToARGBMatrix(const uint8_t* src_uyvy, UYVYToARGBRow = UYVYToARGBRow_LSX; } } +#endif +#if defined(HAS_UYVYTOARGBROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + UYVYToARGBRow = UYVYToARGBRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + UYVYToARGBRow = UYVYToARGBRow_LASX; + } + } #endif for (y = 0; y < height; ++y) { UYVYToARGBRow(src_uyvy, dst_argb, yuvconstants, width); @@ -5466,7 +5506,7 @@ int I422ToRGBAMatrix(const uint8_t* src_y, #if defined(HAS_I422TORGBAROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToRGBARow = I422ToRGBARow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToRGBARow = I422ToRGBARow_LSX; } } @@ -5474,7 +5514,7 @@ int I422ToRGBAMatrix(const uint8_t* src_y, #if defined(HAS_I422TORGBAROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToRGBARow = I422ToRGBARow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToRGBARow = I422ToRGBARow_LASX; } } @@ -5698,7 +5738,7 @@ int I420ToRGBAMatrix(const uint8_t* src_y, #if defined(HAS_I422TORGBAROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToRGBARow = I422ToRGBARow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToRGBARow = I422ToRGBARow_LSX; } } @@ -5706,7 +5746,7 @@ int I420ToRGBAMatrix(const uint8_t* src_y, #if defined(HAS_I422TORGBAROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToRGBARow = I422ToRGBARow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToRGBARow = I422ToRGBARow_LASX; } } @@ -5841,7 +5881,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB24ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToRGB24Row = I422ToRGB24Row_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToRGB24Row = I422ToRGB24Row_LSX; } } @@ -5849,7 +5889,7 @@ int I420ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB24ROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToRGB24Row = I422ToRGB24Row_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToRGB24Row = I422ToRGB24Row_LASX; } } @@ -6072,7 +6112,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB24ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToRGB24Row = I422ToRGB24Row_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToRGB24Row = I422ToRGB24Row_LSX; } } @@ -6080,7 +6120,7 @@ int I422ToRGB24Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB24ROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToRGB24Row = I422ToRGB24Row_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToRGB24Row = I422ToRGB24Row_LASX; } } @@ -6209,7 +6249,7 @@ int I420ToARGB1555(const uint8_t* src_y, #if defined(HAS_I422TOARGB1555ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToARGB1555Row = I422ToARGB1555Row_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToARGB1555Row = I422ToARGB1555Row_LSX; } } @@ -6217,7 +6257,7 @@ int I420ToARGB1555(const uint8_t* src_y, #if defined(HAS_I422TOARGB1555ROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToARGB1555Row = I422ToARGB1555Row_Any_LASX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 16)) { I422ToARGB1555Row = I422ToARGB1555Row_LASX; } } @@ -6308,7 +6348,7 @@ int I420ToARGB4444(const uint8_t* src_y, #if defined(HAS_I422TOARGB4444ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToARGB4444Row = I422ToARGB4444Row_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToARGB4444Row = I422ToARGB4444Row_LSX; } } @@ -6316,7 +6356,7 @@ int I420ToARGB4444(const uint8_t* src_y, #if defined(HAS_I422TOARGB4444ROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToARGB4444Row = I422ToARGB4444Row_Any_LASX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 16)) { I422ToARGB4444Row = I422ToARGB4444Row_LASX; } } @@ -6409,7 +6449,7 @@ int I420ToRGB565Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB565ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToRGB565Row = I422ToRGB565Row_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToRGB565Row = I422ToRGB565Row_LSX; } } @@ -6417,7 +6457,7 @@ int I420ToRGB565Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB565ROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToRGB565Row = I422ToRGB565Row_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToRGB565Row = I422ToRGB565Row_LASX; } } @@ -6568,7 +6608,7 @@ int I422ToRGB565Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB565ROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToRGB565Row = I422ToRGB565Row_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToRGB565Row = I422ToRGB565Row_LSX; } } @@ -6576,7 +6616,7 @@ int I422ToRGB565Matrix(const uint8_t* src_y, #if defined(HAS_I422TORGB565ROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToRGB565Row = I422ToRGB565Row_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToRGB565Row = I422ToRGB565Row_LASX; } } @@ -6701,7 +6741,7 @@ int I420ToRGB565Dither(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { I422ToARGBRow = I422ToARGBRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { I422ToARGBRow = I422ToARGBRow_LSX; } } @@ -6709,7 +6749,7 @@ int I420ToRGB565Dither(const uint8_t* src_y, #if defined(HAS_I422TOARGBROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { I422ToARGBRow = I422ToARGBRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { I422ToARGBRow = I422ToARGBRow_LASX; } } @@ -6759,7 +6799,7 @@ int I420ToRGB565Dither(const uint8_t* src_y, #if defined(HAS_ARGBTORGB565DITHERROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LSX; } } @@ -6767,7 +6807,7 @@ int I420ToRGB565Dither(const uint8_t* src_y, #if defined(HAS_ARGBTORGB565DITHERROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LASX; } } @@ -7971,7 +8011,7 @@ static int I420AlphaToARGBMatrixBilinear( #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -7979,7 +8019,7 @@ static int I420AlphaToARGBMatrixBilinear( #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -8191,7 +8231,7 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -8199,7 +8239,7 @@ static int I422AlphaToARGBMatrixLinear(const uint8_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -8363,7 +8403,7 @@ static int I010AlphaToARGBMatrixBilinear( #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -8371,7 +8411,7 @@ static int I010AlphaToARGBMatrixBilinear( #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -8557,7 +8597,7 @@ static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -8565,7 +8605,7 @@ static int I210AlphaToARGBMatrixLinear(const uint16_t* src_y, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } diff --git a/source/convert_from_argb.cc b/source/convert_from_argb.cc index 09a07dc81..2c7ec92ce 100644 --- a/source/convert_from_argb.cc +++ b/source/convert_from_argb.cc @@ -113,7 +113,7 @@ int ARGBToI444Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -121,7 +121,7 @@ int ARGBToI444Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -305,7 +305,7 @@ int ARGBToI422Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -313,7 +313,7 @@ int ARGBToI422Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -504,7 +504,7 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -512,7 +512,7 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -652,6 +652,14 @@ int ARGBToNV12Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -778,7 +786,7 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -786,7 +794,7 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -931,6 +939,14 @@ int ARGBToNV16Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -1051,7 +1067,7 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -1059,7 +1075,7 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -1193,6 +1209,14 @@ int ARGBToNV24Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -1301,7 +1325,7 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -1309,7 +1333,7 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -1449,6 +1473,14 @@ int ARGBToNV21Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -2008,7 +2040,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -2016,7 +2048,7 @@ int ARGBToP010Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -2247,6 +2279,14 @@ int ARGBToP010Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -2406,7 +2446,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -2414,7 +2454,7 @@ int ARGBToP210Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -2645,6 +2685,14 @@ int ARGBToP210Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -2789,7 +2837,7 @@ int ARGBToP410Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -2797,7 +2845,7 @@ int ARGBToP410Matrix(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -3017,6 +3065,14 @@ int ARGBToP410Matrix(const uint8_t* src_argb, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -3429,7 +3485,7 @@ int ARGBToRGB565Dither(const uint8_t* src_argb, #if defined(HAS_ARGBTORGB565DITHERROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LSX; } } @@ -3437,7 +3493,7 @@ int ARGBToRGB565Dither(const uint8_t* src_argb, #if defined(HAS_ARGBTORGB565DITHERROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToRGB565DitherRow = ARGBToRGB565DitherRow_LASX; } } @@ -3517,6 +3573,14 @@ int ARGBToRGB565(const uint8_t* src_argb, } } #endif +#if defined(HAS_ARGBTORGB565ROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBToRGB565Row = ARGBToRGB565Row_Any_LASX; + if (IS_ALIGNED(width, 16)) { + ARGBToRGB565Row = ARGBToRGB565Row_LASX; + } + } +#endif #if defined(HAS_ARGBTORGB565ROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { @@ -4258,6 +4322,14 @@ static int RAWToNVMatrix(const uint8_t* src_raw, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(halfwidth, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; diff --git a/source/planar_functions.cc b/source/planar_functions.cc index 0590febb9..afd7a9cc9 100644 --- a/source/planar_functions.cc +++ b/source/planar_functions.cc @@ -97,12 +97,12 @@ void CopyPlane(const uint8_t* src_y, #endif #if defined(HAS_COPYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { - CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LSX : CopyRow_Any_LSX; + CopyRow = IS_ALIGNED(width, 16) ? CopyRow_LSX : CopyRow_Any_LSX; } #endif #if defined(HAS_COPYROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { - CopyRow = IS_ALIGNED(width, 64) ? CopyRow_LASX : CopyRow_Any_LASX; + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LASX : CopyRow_Any_LASX; } #endif @@ -700,11 +700,19 @@ void SplitUVPlane(const uint8_t* src_uv, #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { SplitUVRow = SplitUVRow_LSX; } } #endif +#if defined(HAS_SPLITUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitUVRow = SplitUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + SplitUVRow = SplitUVRow_LASX; + } + } +#endif #if defined(HAS_SPLITUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { SplitUVRow = SplitUVRow_RVV; @@ -798,6 +806,14 @@ void MergeUVPlane(const uint8_t* src_u, } } #endif +#if defined(HAS_MERGEUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + MergeUVRow = MergeUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + MergeUVRow = MergeUVRow_LASX; + } + } +#endif #if defined(HAS_MERGEUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { MergeUVRow = MergeUVRow_RVV; @@ -2661,6 +2677,14 @@ int UYVYToY(const uint8_t* src_uyvy, } } #endif +#if defined(HAS_UYVYTOYROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + UYVYToYRow = UYVYToYRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + UYVYToYRow = UYVYToYRow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { UYVYToYRow(src_uyvy, dst_y, width); @@ -2725,7 +2749,7 @@ void MirrorPlane(const uint8_t* src_y, #if defined(HAS_MIRRORROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { MirrorRow = MirrorRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { MirrorRow = MirrorRow_LSX; } } @@ -2733,7 +2757,7 @@ void MirrorPlane(const uint8_t* src_y, #if defined(HAS_MIRRORROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { MirrorRow = MirrorRow_Any_LASX; - if (IS_ALIGNED(width, 64)) { + if (IS_ALIGNED(width, 32)) { MirrorRow = MirrorRow_LASX; } } @@ -3002,7 +3026,7 @@ int ARGBMirror(const uint8_t* src_argb, #if defined(HAS_ARGBMIRRORROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBMirrorRow = ARGBMirrorRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBMirrorRow = ARGBMirrorRow_LSX; } } @@ -3010,7 +3034,7 @@ int ARGBMirror(const uint8_t* src_argb, #if defined(HAS_ARGBMIRRORROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBMirrorRow = ARGBMirrorRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBMirrorRow = ARGBMirrorRow_LASX; } } @@ -3141,11 +3165,19 @@ int ARGBBlend(const uint8_t* src_argb0, #if defined(HAS_ARGBBLENDROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBBlendRow = ARGBBlendRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBBlendRow = ARGBBlendRow_LSX; } } #endif +#if defined(HAS_ARGBBLENDROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBBlendRow = ARGBBlendRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBBlendRow = ARGBBlendRow_LASX; + } + } +#endif #if defined(HAS_ARGBBLENDROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { ARGBBlendRow = ARGBBlendRow_RVV; @@ -3855,6 +3887,14 @@ void SetPlane(uint8_t* dst_y, } } #endif +#if defined(HAS_SETROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SetRow = SetRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + SetRow = SetRow_LASX; + } + } +#endif // Set plane for (y = 0; y < height; ++y) { @@ -3946,6 +3986,14 @@ int ARGBRect(uint8_t* dst_argb, } } #endif +#if defined(HAS_ARGBSETROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBSetRow = ARGBSetRow_Any_LASX; + if (IS_ALIGNED(width, 8)) { + ARGBSetRow = ARGBSetRow_LASX; + } + } +#endif // Set plane for (y = 0; y < height; ++y) { @@ -4021,7 +4069,7 @@ int ARGBAttenuate(const uint8_t* src_argb, #if defined(HAS_ARGBATTENUATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBAttenuateRow = ARGBAttenuateRow_LSX; } } @@ -4029,7 +4077,7 @@ int ARGBAttenuate(const uint8_t* src_argb, #if defined(HAS_ARGBATTENUATEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBAttenuateRow = ARGBAttenuateRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBAttenuateRow = ARGBAttenuateRow_LASX; } } @@ -4315,9 +4363,14 @@ int ARGBColorMatrix(const uint8_t* src_argb, } #endif #if defined(HAS_ARGBCOLORMATRIXROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) { + if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) { ARGBColorMatrixRow = ARGBColorMatrixRow_LSX; } +#endif +#if defined(HAS_ARGBCOLORMATRIXROW_LASX) + if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) { + ARGBColorMatrixRow = ARGBColorMatrixRow_LASX; + } #endif for (y = 0; y < height; ++y) { ARGBColorMatrixRow(src_argb, dst_argb, matrix_argb, width); @@ -4480,9 +4533,14 @@ int ARGBQuantize(uint8_t* dst_argb, } #endif #if defined(HAS_ARGBQUANTIZEROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 8)) { + if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 4)) { ARGBQuantizeRow = ARGBQuantizeRow_LSX; } +#endif +#if defined(HAS_ARGBQUANTIZEROW_LASX) + if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 8)) { + ARGBQuantizeRow = ARGBQuantizeRow_LASX; + } #endif for (y = 0; y < height; ++y) { ARGBQuantizeRow(dst, scale, interval_size, interval_offset, width); @@ -4783,11 +4841,19 @@ int InterpolatePlane(const uint8_t* src0, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { InterpolateRow = InterpolateRow_LSX; } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; @@ -5016,7 +5082,7 @@ int ARGBShuffle(const uint8_t* src_argb, #if defined(HAS_ARGBSHUFFLEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBShuffleRow = ARGBShuffleRow_Any_LSX; - if (IS_ALIGNED(width, 8)) { + if (IS_ALIGNED(width, 4)) { ARGBShuffleRow = ARGBShuffleRow_LSX; } } @@ -5024,7 +5090,7 @@ int ARGBShuffle(const uint8_t* src_argb, #if defined(HAS_ARGBSHUFFLEROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBShuffleRow = ARGBShuffleRow_Any_LASX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBShuffleRow = ARGBShuffleRow_LASX; } } @@ -5278,7 +5344,7 @@ static int ARGBSobelize(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LSX; } } @@ -5286,7 +5352,7 @@ static int ARGBSobelize(const uint8_t* src_argb, #if defined(HAS_ARGBTOYMATRIXROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { ARGBToYMatrixRow = ARGBToYMatrixRow_Any_LASX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { ARGBToYMatrixRow = ARGBToYMatrixRow_LASX; } } @@ -5404,10 +5470,18 @@ int ARGBSobel(const uint8_t* src_argb, #if defined(HAS_SOBELROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SobelRow = SobelRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { SobelRow = SobelRow_LSX; } } +#endif +#if defined(HAS_SOBELROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SobelRow = SobelRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + SobelRow = SobelRow_LASX; + } + } #endif return ARGBSobelize(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width, height, SobelRow); @@ -5442,10 +5516,18 @@ int ARGBSobelToPlane(const uint8_t* src_argb, #if defined(HAS_SOBELTOPLANEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SobelToPlaneRow = SobelToPlaneRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { SobelToPlaneRow = SobelToPlaneRow_LSX; } } +#endif +#if defined(HAS_SOBELTOPLANEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SobelToPlaneRow = SobelToPlaneRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + SobelToPlaneRow = SobelToPlaneRow_LASX; + } + } #endif return ARGBSobelize(src_argb, src_stride_argb, dst_y, dst_stride_y, width, height, SobelToPlaneRow); @@ -5481,10 +5563,18 @@ int ARGBSobelXY(const uint8_t* src_argb, #if defined(HAS_SOBELXYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SobelXYRow = SobelXYRow_Any_LSX; - if (IS_ALIGNED(width, 16)) { + if (IS_ALIGNED(width, 8)) { SobelXYRow = SobelXYRow_LSX; } } +#endif +#if defined(HAS_SOBELXYROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SobelXYRow = SobelXYRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + SobelXYRow = SobelXYRow_LASX; + } + } #endif return ARGBSobelize(src_argb, src_stride_argb, dst_argb, dst_stride_argb, width, height, SobelXYRow); @@ -5611,11 +5701,19 @@ int HalfFloatPlane(const uint16_t* src_y, #if defined(HAS_HALFFLOATROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { HalfFloatRow = HalfFloatRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 8)) { HalfFloatRow = HalfFloatRow_LSX; } } #endif +#if defined(HAS_HALFFLOATROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + HalfFloatRow = HalfFloatRow_Any_LASX; + if (IS_ALIGNED(width, 16)) { + HalfFloatRow = HalfFloatRow_LASX; + } + } +#endif for (y = 0; y < height; ++y) { HalfFloatRow(src_y, dst_y, scale, width); @@ -5794,8 +5892,14 @@ int ARGBExtractAlpha(const uint8_t* src_argb, #endif #if defined(HAS_ARGBEXTRACTALPHAROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { - ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_LSX - : ARGBExtractAlphaRow_Any_LSX; + ARGBExtractAlphaRow = IS_ALIGNED(width, 8) ? ARGBExtractAlphaRow_LSX + : ARGBExtractAlphaRow_Any_LSX; + } +#endif +#if defined(HAS_ARGBEXTRACTALPHAROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + ARGBExtractAlphaRow = IS_ALIGNED(width, 16) ? ARGBExtractAlphaRow_LASX + : ARGBExtractAlphaRow_Any_LASX; } #endif #if defined(HAS_ARGBEXTRACTALPHAROW_WASMSIMD) @@ -6065,11 +6169,19 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_SPLITUVROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { SplitUVRow = SplitUVRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { SplitUVRow = SplitUVRow_LSX; } } #endif +#if defined(HAS_SPLITUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + SplitUVRow = SplitUVRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + SplitUVRow = SplitUVRow_LASX; + } + } +#endif #if defined(HAS_SPLITUVROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { SplitUVRow = SplitUVRow_RVV; @@ -6110,11 +6222,19 @@ int UYVYToNV12(const uint8_t* src_uyvy, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { InterpolateRow = InterpolateRow_LSX; } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; diff --git a/source/rotate.cc b/source/rotate.cc index c155a0d90..d0c4696be 100644 --- a/source/rotate.cc +++ b/source/rotate.cc @@ -37,14 +37,14 @@ void TransposePlane(const uint8_t* src, int dst_stride, int width, int height) = NULL; #endif #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \ - defined(HAS_TRANSPOSEWX16_WASMSIMD) + defined(HAS_TRANSPOSEWX16_LASX) || defined(HAS_TRANSPOSEWX16_WASMSIMD) void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = NULL; -#elif defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) +#elif defined(HAS_TRANSPOSEWX16_NEON) void (*TransposeWx16)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = TransposeWx16_C; #endif -#if !defined(HAS_TRANSPOSEWX16_LSX) && !defined(HAS_TRANSPOSEWX16_NEON) +#if !defined(HAS_TRANSPOSEWX16_NEON) void (*TransposeWx8)(const uint8_t* src, int src_stride, uint8_t* dst, int dst_stride, int width) = TransposeWx8_C; #endif @@ -91,11 +91,19 @@ void TransposePlane(const uint8_t* src, TransposeWx16 = TransposeWx16_AVX512BW; } #endif -#if defined(HAS_TRANSPOSEWX16_LSX) +#if defined(HAS_TRANSPOSEWX8_LSX) if (TestCpuFlag(kCpuHasLSX)) { - TransposeWx16 = TransposeWx16_Any_LSX; + TransposeWx8 = TransposeWx8_Any_LSX; if (IS_ALIGNED(width, 16)) { - TransposeWx16 = TransposeWx16_LSX; + TransposeWx8 = TransposeWx8_LSX; + } + } +#endif +#if defined(HAS_TRANSPOSEWX16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + TransposeWx16 = TransposeWx16_Any_LASX; + if (IS_ALIGNED(width, 16)) { + TransposeWx16 = TransposeWx16_LASX; } } #endif @@ -123,7 +131,7 @@ void TransposePlane(const uint8_t* src, } #endif #if defined(HAS_TRANSPOSEWX16_AVX2) || defined(HAS_TRANSPOSEWX16_AVX512BW) || \ - defined(HAS_TRANSPOSEWX16_LSX) || defined(HAS_TRANSPOSEWX16_NEON) || \ + defined(HAS_TRANSPOSEWX16_LASX) || defined(HAS_TRANSPOSEWX16_NEON) || \ defined(HAS_TRANSPOSEWX16_WASMSIMD) // Work across the source in 16x16 tiles if (TransposeWx16) { @@ -135,7 +143,7 @@ void TransposePlane(const uint8_t* src, } } #endif -#if !defined(HAS_TRANSPOSEWX16_LSX) && !defined(HAS_TRANSPOSEWX16_NEON) +#if !defined(HAS_TRANSPOSEWX16_NEON) // Work across the source in 8x8 tiles while (i >= 8) { TransposeWx8(src, src_stride, dst, dst_stride, width); @@ -225,7 +233,7 @@ void RotatePlane180(const uint8_t* src, #if defined(HAS_MIRRORROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { MirrorRow = MirrorRow_Any_LSX; - if (IS_ALIGNED(width, 32)) { + if (IS_ALIGNED(width, 16)) { MirrorRow = MirrorRow_LSX; } } @@ -233,7 +241,7 @@ void RotatePlane180(const uint8_t* src, #if defined(HAS_MIRRORROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { MirrorRow = MirrorRow_Any_LASX; - if (IS_ALIGNED(width, 64)) { + if (IS_ALIGNED(width, 32)) { MirrorRow = MirrorRow_LASX; } } @@ -288,12 +296,12 @@ void RotatePlane180(const uint8_t* src, #endif #if defined(HAS_COPYROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { - CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LSX : CopyRow_Any_LSX; + CopyRow = IS_ALIGNED(width, 16) ? CopyRow_LSX : CopyRow_Any_LSX; } #endif #if defined(HAS_COPYROW_LASX) if (TestCpuFlag(kCpuHasLASX)) { - CopyRow = IS_ALIGNED(width, 64) ? CopyRow_LASX : CopyRow_Any_LASX; + CopyRow = IS_ALIGNED(width, 32) ? CopyRow_LASX : CopyRow_Any_LASX; } #endif @@ -327,26 +335,28 @@ void SplitTransposeUV(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEUVWX16_AVX2) || \ defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \ - defined(HAS_TRANSPOSEUVWX16_WASMSIMD) + defined(HAS_TRANSPOSEUVWX16_LASX) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD) void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width) = NULL; -#elif defined(HAS_TRANSPOSEUVWX16_LSX) - void (*TransposeUVWx16)(const uint8_t* src, int src_stride, uint8_t* dst_a, - int dst_stride_a, uint8_t* dst_b, int dst_stride_b, - int width) = TransposeUVWx16_C; #endif -#if !defined(HAS_TRANSPOSEUVWX16_LSX) void (*TransposeUVWx8)(const uint8_t* src, int src_stride, uint8_t* dst_a, int dst_stride_a, uint8_t* dst_b, int dst_stride_b, int width) = TransposeUVWx8_C; -#endif -#if defined(HAS_TRANSPOSEUVWX16_LSX) +#if defined(HAS_TRANSPOSEUVWX8_LSX) if (TestCpuFlag(kCpuHasLSX)) { - TransposeUVWx16 = TransposeUVWx16_Any_LSX; + TransposeUVWx8 = TransposeUVWx8_Any_LSX; if (IS_ALIGNED(width, 8)) { - TransposeUVWx16 = TransposeUVWx16_LSX; + TransposeUVWx8 = TransposeUVWx8_LSX; + } + } +#endif +#if defined(HAS_TRANSPOSEUVWX16_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + TransposeUVWx16 = TransposeUVWx16_Any_LASX; + if (IS_ALIGNED(width, 8)) { + TransposeUVWx16 = TransposeUVWx16_LASX; } } #endif @@ -410,7 +420,7 @@ void SplitTransposeUV(const uint8_t* src, #endif #if defined(HAS_TRANSPOSEUVWX16_AVX2) || \ defined(HAS_TRANSPOSEUVWX16_AVX512BW) || \ - defined(HAS_TRANSPOSEUVWX16_LSX) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD) + defined(HAS_TRANSPOSEUVWX16_LASX) || defined(HAS_TRANSPOSEUVWX16_WASMSIMD) // Work through the source in 16x16 tiles. if (TransposeUVWx16) { while (i >= 16) { @@ -423,7 +433,6 @@ void SplitTransposeUV(const uint8_t* src, } } #endif -#if !defined(HAS_TRANSPOSEUVWX16_LSX) // Work through the source in 8x8 tiles. while (i >= 8) { TransposeUVWx8(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, @@ -433,7 +442,6 @@ void SplitTransposeUV(const uint8_t* src, dst_b += 8; // Move over 8 columns. i -= 8; } -#endif if (i > 0) { TransposeUVWxH_C(src, src_stride, dst_a, dst_stride_a, dst_b, dst_stride_b, @@ -504,10 +512,15 @@ void SplitRotateUV180(const uint8_t* src, } #endif #if defined(HAS_MIRRORSPLITUVROW_LSX) - if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 32)) { + if (TestCpuFlag(kCpuHasLSX) && IS_ALIGNED(width, 16)) { MirrorSplitUVRow = MirrorSplitUVRow_LSX; } #endif +#if defined(HAS_MIRRORSPLITUVROW_LASX) + if (TestCpuFlag(kCpuHasLASX) && IS_ALIGNED(width, 32)) { + MirrorSplitUVRow = MirrorSplitUVRow_LASX; + } +#endif #if defined(HAS_MIRRORSPLITUVROW_WASMSIMD) if (TestCpuFlag(kCpuHasWASMSIMD) && IS_ALIGNED(width, 16)) { MirrorSplitUVRow = MirrorSplitUVRow_WASMSIMD; diff --git a/source/rotate_any.cc b/source/rotate_any.cc index 70f8367ae..499047cb4 100644 --- a/source/rotate_any.cc +++ b/source/rotate_any.cc @@ -40,8 +40,11 @@ TANY(TransposeWx8_Any_SSSE3, TransposeWx8_SSSE3, TransposeWx8_C, 15) #ifdef HAS_TRANSPOSEWX16_AVX2 TANY(TransposeWx16_Any_AVX2, TransposeWx16_AVX2, TransposeWx16_C, 15) #endif -#ifdef HAS_TRANSPOSEWX16_LSX -TANY(TransposeWx16_Any_LSX, TransposeWx16_LSX, TransposeWx16_C, 15) +#ifdef HAS_TRANSPOSEWX8_LSX +TANY(TransposeWx8_Any_LSX, TransposeWx8_LSX, TransposeWx8_C, 15) +#endif +#ifdef HAS_TRANSPOSEWX16_LASX +TANY(TransposeWx16_Any_LASX, TransposeWx16_LASX, TransposeWx16_C, 15) #endif #ifdef HAS_TRANSPOSEWX8_WASMSIMD TANY(TransposeWx8_Any_WASMSIMD, TransposeWx8_WASMSIMD, TransposeWx8_C, 15) @@ -74,8 +77,11 @@ TUVANY(TransposeUVWx8_Any_SSE2, TransposeUVWx8_SSE2, TransposeUVWx8_C, 7) #ifdef HAS_TRANSPOSEUVWX16_AVX2 TUVANY(TransposeUVWx16_Any_AVX2, TransposeUVWx16_AVX2, TransposeUVWx16_C, 7) #endif -#ifdef HAS_TRANSPOSEUVWX16_LSX -TUVANY(TransposeUVWx16_Any_LSX, TransposeUVWx16_LSX, TransposeUVWx16_C, 7) +#ifdef HAS_TRANSPOSEUVWX8_LSX +TUVANY(TransposeUVWx8_Any_LSX, TransposeUVWx8_LSX, TransposeUVWx8_C, 7) +#endif +#ifdef HAS_TRANSPOSEUVWX16_LASX +TUVANY(TransposeUVWx16_Any_LASX, TransposeUVWx16_LASX, TransposeUVWx16_C, 7) #endif #ifdef HAS_TRANSPOSEUVWX8_WASMSIMD TUVANY(TransposeUVWx8_Any_WASMSIMD, diff --git a/source/rotate_lasx.cc b/source/rotate_lasx.cc index 9b877445b..8a50cbe2a 100644 --- a/source/rotate_lasx.cc +++ b/source/rotate_lasx.cc @@ -11,15 +11,307 @@ #include "libyuv/rotate_row.h" #if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) -#include "libyuv/loongson_intrinsics.h" #ifdef __cplusplus namespace libyuv { extern "C" { #endif -// Transpose 32 bit values (ARGB) in 2x 4x4 tiles per step (4 cols x 8 rows -> -// 8 cols x 4 rows). Width is a multiple of 8. +void TransposeWx16_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr8, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr9, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr10, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr11, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr12, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr13, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr14, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr15, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" + + "xvpermi.q $xr0, $xr8, 0x02 \n" + "xvpermi.q $xr1, $xr9, 0x02 \n" + "xvpermi.q $xr2, $xr10, 0x02 \n" + "xvpermi.q $xr3, $xr11, 0x02 \n" + "xvpermi.q $xr4, $xr12, 0x02 \n" + "xvpermi.q $xr5, $xr13, 0x02 \n" + "xvpermi.q $xr6, $xr14, 0x02 \n" + "xvpermi.q $xr7, $xr15, 0x02 \n" + + "xvilvl.b $xr8, $xr1, $xr0 \n" + "xvilvh.b $xr9, $xr1, $xr0 \n" + "xvilvl.b $xr10, $xr3, $xr2 \n" + "xvilvh.b $xr11, $xr3, $xr2 \n" + "xvilvl.b $xr12, $xr5, $xr4 \n" + "xvilvh.b $xr13, $xr5, $xr4 \n" + "xvilvl.b $xr14, $xr7, $xr6 \n" + "xvilvh.b $xr15, $xr7, $xr6 \n" + + "xvilvl.h $xr0, $xr10, $xr8 \n" + "xvilvh.h $xr1, $xr10, $xr8 \n" + "xvilvl.h $xr2, $xr11, $xr9 \n" + "xvilvh.h $xr3, $xr11, $xr9 \n" + "xvilvl.h $xr4, $xr14, $xr12 \n" + "xvilvh.h $xr5, $xr14, $xr12 \n" + "xvilvl.h $xr6, $xr15, $xr13 \n" + "xvilvh.h $xr7, $xr15, $xr13 \n" + + "xvilvl.w $xr8, $xr4, $xr0 \n" + "xvilvh.w $xr9, $xr4, $xr0 \n" + "xvilvl.w $xr10, $xr5, $xr1 \n" + "xvilvh.w $xr11, $xr5, $xr1 \n" + "xvilvl.w $xr12, $xr6, $xr2 \n" + "xvilvh.w $xr13, $xr6, $xr2 \n" + "xvilvl.w $xr14, $xr7, $xr3 \n" + "xvilvh.w $xr15, $xr7, $xr3 \n" + + "xvpermi.d $xr0, $xr8, 0x08 \n" + "xvpermi.d $xr1, $xr8, 0x0d \n" + "xvpermi.d $xr2, $xr9, 0x08 \n" + "xvpermi.d $xr3, $xr9, 0x0d \n" + "xvpermi.d $xr4, $xr10, 0x08 \n" + "xvpermi.d $xr5, $xr10, 0x0d \n" + "xvpermi.d $xr6, $xr11, 0x08 \n" + "xvpermi.d $xr7, $xr11, 0x0d \n" + "xvpermi.d $xr8, $xr12, 0x08 \n" + "xvpermi.d $xr9, $xr12, 0x0d \n" + "xvpermi.d $xr10, $xr13, 0x08 \n" + "xvpermi.d $xr11, $xr13, 0x0d \n" + "xvpermi.d $xr12, $xr14, 0x08 \n" + "xvpermi.d $xr13, $xr14, 0x0d \n" + "xvpermi.d $xr14, $xr15, 0x08 \n" + "xvpermi.d $xr15, $xr15, 0x0d \n" + + "vst $vr0, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr1, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr2, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr3, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr4, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr5, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr6, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr7, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr8, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr9, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr10, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr11, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr12, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr13, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr14, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr15, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15"); +} + +void TransposeUVWx16_LASX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t da = dst_stride_a; + const ptrdiff_t db = dst_stride_b; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr8, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr9, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr10, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr11, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr12, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr13, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr14, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr15, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" + + "xvpermi.q $xr0, $xr8, 0x02 \n" + "xvpermi.q $xr1, $xr9, 0x02 \n" + "xvpermi.q $xr2, $xr10, 0x02 \n" + "xvpermi.q $xr3, $xr11, 0x02 \n" + "xvpermi.q $xr4, $xr12, 0x02 \n" + "xvpermi.q $xr5, $xr13, 0x02 \n" + "xvpermi.q $xr6, $xr14, 0x02 \n" + "xvpermi.q $xr7, $xr15, 0x02 \n" + + "xvilvl.b $xr8, $xr1, $xr0 \n" + "xvilvh.b $xr9, $xr1, $xr0 \n" + "xvilvl.b $xr10, $xr3, $xr2 \n" + "xvilvh.b $xr11, $xr3, $xr2 \n" + "xvilvl.b $xr12, $xr5, $xr4 \n" + "xvilvh.b $xr13, $xr5, $xr4 \n" + "xvilvl.b $xr14, $xr7, $xr6 \n" + "xvilvh.b $xr15, $xr7, $xr6 \n" + + "xvilvl.h $xr0, $xr10, $xr8 \n" + "xvilvh.h $xr1, $xr10, $xr8 \n" + "xvilvl.h $xr2, $xr11, $xr9 \n" + "xvilvh.h $xr3, $xr11, $xr9 \n" + "xvilvl.h $xr4, $xr14, $xr12 \n" + "xvilvh.h $xr5, $xr14, $xr12 \n" + "xvilvl.h $xr6, $xr15, $xr13 \n" + "xvilvh.h $xr7, $xr15, $xr13 \n" + + "xvilvl.w $xr8, $xr4, $xr0 \n" + "xvilvh.w $xr9, $xr4, $xr0 \n" + "xvilvl.w $xr10, $xr5, $xr1 \n" + "xvilvh.w $xr11, $xr5, $xr1 \n" + "xvilvl.w $xr12, $xr6, $xr2 \n" + "xvilvh.w $xr13, $xr6, $xr2 \n" + "xvilvl.w $xr14, $xr7, $xr3 \n" + "xvilvh.w $xr15, $xr7, $xr3 \n" + + "xvpermi.d $xr0, $xr8, 0x08 \n" + "xvpermi.d $xr1, $xr8, 0x0d \n" + "xvpermi.d $xr2, $xr9, 0x08 \n" + "xvpermi.d $xr3, $xr9, 0x0d \n" + "xvpermi.d $xr4, $xr10, 0x08 \n" + "xvpermi.d $xr5, $xr10, 0x0d \n" + "xvpermi.d $xr6, $xr11, 0x08 \n" + "xvpermi.d $xr7, $xr11, 0x0d \n" + "xvpermi.d $xr8, $xr12, 0x08 \n" + "xvpermi.d $xr9, $xr12, 0x0d \n" + "xvpermi.d $xr10, $xr13, 0x08 \n" + "xvpermi.d $xr11, $xr13, 0x0d \n" + "xvpermi.d $xr12, $xr14, 0x08 \n" + "xvpermi.d $xr13, $xr14, 0x0d \n" + "xvpermi.d $xr14, $xr15, 0x08 \n" + "xvpermi.d $xr15, $xr15, 0x0d \n" + + "vst $vr0, %[dst_a], 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vst $vr1, %[dst_b], 0 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vst $vr2, %[dst_a], 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vst $vr3, %[dst_b], 0 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vst $vr4, %[dst_a], 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vst $vr5, %[dst_b], 0 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vst $vr6, %[dst_a], 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vst $vr7, %[dst_b], 0 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vst $vr8, %[dst_a], 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vst $vr9, %[dst_b], 0 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vst $vr10, %[dst_a], 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vst $vr11, %[dst_b], 0 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vst $vr12, %[dst_a], 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vst $vr13, %[dst_b], 0 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vst $vr14, %[dst_a], 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vst $vr15, %[dst_b], 0 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst_a] "+r"(dst_a), // %[dst_a] + [dst_b] "+r"(dst_b), // %[dst_b] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [da] "r"(da), // %[da] + [db] "r"(db) // %[db] + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15"); +} + void TransposeWx4_32_LASX(const uint8_t* src, int src_stride, uint8_t* dst, @@ -50,25 +342,21 @@ void TransposeWx4_32_LASX(const uint8_t* src, "vldx $vr7, %[src4], %[s3] \n" "add.d %[src], %[src4], %[s4] \n" - // Combine rows (0,4), (1,5), (2,6), (3,7) into 256-bit xr0..xr3 "xvpermi.q $xr0, $xr4, 0x02 \n" "xvpermi.q $xr1, $xr5, 0x02 \n" "xvpermi.q $xr2, $xr6, 0x02 \n" "xvpermi.q $xr3, $xr7, 0x02 \n" - // Transpose 2x2 across both 128-bit lanes "xvilvl.w $xr4, $xr1, $xr0 \n" "xvilvh.w $xr5, $xr1, $xr0 \n" "xvilvl.w $xr6, $xr3, $xr2 \n" "xvilvh.w $xr7, $xr3, $xr2 \n" - // Transpose 4x4 across both 128-bit lanes "xvilvl.d $xr0, $xr6, $xr4 \n" "xvilvh.d $xr1, $xr6, $xr4 \n" "xvilvl.d $xr2, $xr7, $xr5 \n" "xvilvh.d $xr3, $xr7, $xr5 \n" - // Store 4 full 256-bit (32-byte) destination rows "xvst $xr0, %[dst0], 0 \n" "xvst $xr1, %[dst1], 0 \n" "xvst $xr2, %[dst2], 0 \n" diff --git a/source/rotate_lsx.cc b/source/rotate_lsx.cc index b2580562a..531145bf1 100644 --- a/source/rotate_lsx.cc +++ b/source/rotate_lsx.cc @@ -1,8 +1,6 @@ /* * Copyright 2022 The LibYuv Project Authors. All rights reserved. * - * Copyright (c) 2022 Loongson Technology Corporation Limited - * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found @@ -13,203 +11,221 @@ #include "libyuv/rotate_row.h" #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) -#include "libyuv/loongson_intrinsics.h" #ifdef __cplusplus namespace libyuv { extern "C" { #endif -#define ILVLH_B(in0, in1, in2, in3, out0, out1, out2, out3) \ - { \ - DUP2_ARG2(__lsx_vilvl_b, in1, in0, in3, in2, out0, out2); \ - DUP2_ARG2(__lsx_vilvh_b, in1, in0, in3, in2, out1, out3); \ - } +void TransposeWx8_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst, + int dst_stride, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t d = dst_stride; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" -#define ILVLH_H(in0, in1, in2, in3, out0, out1, out2, out3) \ - { \ - DUP2_ARG2(__lsx_vilvl_h, in1, in0, in3, in2, out0, out2); \ - DUP2_ARG2(__lsx_vilvh_h, in1, in0, in3, in2, out1, out3); \ - } + "vilvl.b $vr8, $vr1, $vr0 \n" + "vilvh.b $vr9, $vr1, $vr0 \n" + "vilvl.b $vr10, $vr3, $vr2 \n" + "vilvh.b $vr11, $vr3, $vr2 \n" + "vilvl.b $vr12, $vr5, $vr4 \n" + "vilvh.b $vr13, $vr5, $vr4 \n" + "vilvl.b $vr14, $vr7, $vr6 \n" + "vilvh.b $vr15, $vr7, $vr6 \n" -#define ILVLH_W(in0, in1, in2, in3, out0, out1, out2, out3) \ - { \ - DUP2_ARG2(__lsx_vilvl_w, in1, in0, in3, in2, out0, out2); \ - DUP2_ARG2(__lsx_vilvh_w, in1, in0, in3, in2, out1, out3); \ - } + "vilvl.h $vr0, $vr10, $vr8 \n" + "vilvh.h $vr1, $vr10, $vr8 \n" + "vilvl.h $vr2, $vr11, $vr9 \n" + "vilvh.h $vr3, $vr11, $vr9 \n" + "vilvl.h $vr4, $vr14, $vr12 \n" + "vilvh.h $vr5, $vr14, $vr12 \n" + "vilvl.h $vr6, $vr15, $vr13 \n" + "vilvh.h $vr7, $vr15, $vr13 \n" -#define ILVLH_D(in0, in1, in2, in3, out0, out1, out2, out3) \ - { \ - DUP2_ARG2(__lsx_vilvl_d, in1, in0, in3, in2, out0, out2); \ - DUP2_ARG2(__lsx_vilvh_d, in1, in0, in3, in2, out1, out3); \ - } + "vilvl.w $vr8, $vr4, $vr0 \n" + "vilvh.w $vr9, $vr4, $vr0 \n" + "vilvl.w $vr10, $vr5, $vr1 \n" + "vilvh.w $vr11, $vr5, $vr1 \n" + "vilvl.w $vr12, $vr6, $vr2 \n" + "vilvh.w $vr13, $vr6, $vr2 \n" + "vilvl.w $vr14, $vr7, $vr3 \n" + "vilvh.w $vr15, $vr7, $vr3 \n" -#define LSX_ST_4(_dst0, _dst1, _dst2, _dst3, _dst, _stride, _stride2, \ - _stride3, _stride4) \ - { \ - __lsx_vst(_dst0, _dst, 0); \ - __lsx_vstx(_dst1, _dst, _stride); \ - __lsx_vstx(_dst2, _dst, _stride2); \ - __lsx_vstx(_dst3, _dst, _stride3); \ - _dst += _stride4; \ - } - -#define LSX_ST_2(_dst0, _dst1, _dst, _stride, _stride2) \ - { \ - __lsx_vst(_dst0, _dst, 0); \ - __lsx_vstx(_dst1, _dst, _stride); \ - _dst += _stride2; \ - } - -void TransposeWx16_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst, - int dst_stride, - int width) { - int x; - int len = width / 16; - uint8_t* s; - int src_stride2 = src_stride << 1; - int src_stride3 = src_stride + src_stride2; - int src_stride4 = src_stride2 << 1; - int dst_stride2 = dst_stride << 1; - int dst_stride3 = dst_stride + dst_stride2; - int dst_stride4 = dst_stride2 << 1; - __m128i src0, src1, src2, src3, dst0, dst1, dst2, dst3; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; - __m128i res0, res1, res2, res3, res4, res5, res6, res7, res8, res9; - - for (x = 0; x < len; x++) { - s = (uint8_t*)src; - src0 = __lsx_vld(s, 0); - src1 = __lsx_vldx(s, src_stride); - src2 = __lsx_vldx(s, src_stride2); - src3 = __lsx_vldx(s, src_stride3); - s += src_stride4; - ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); - ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3); - src0 = __lsx_vld(s, 0); - src1 = __lsx_vldx(s, src_stride); - src2 = __lsx_vldx(s, src_stride2); - src3 = __lsx_vldx(s, src_stride3); - s += src_stride4; - ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); - ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7); - ILVLH_W(reg0, reg4, reg1, reg5, res0, res1, res2, res3); - ILVLH_W(reg2, reg6, reg3, reg7, res4, res5, res6, res7); - src0 = __lsx_vld(s, 0); - src1 = __lsx_vldx(s, src_stride); - src2 = __lsx_vldx(s, src_stride2); - src3 = __lsx_vldx(s, src_stride3); - s += src_stride4; - ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); - ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3); - src0 = __lsx_vld(s, 0); - src1 = __lsx_vldx(s, src_stride); - src2 = __lsx_vldx(s, src_stride2); - src3 = __lsx_vldx(s, src_stride3); - s += src_stride4; - ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); - ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7); - res8 = __lsx_vilvl_w(reg4, reg0); - res9 = __lsx_vilvh_w(reg4, reg0); - ILVLH_D(res0, res8, res1, res9, dst0, dst1, dst2, dst3); - LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3, - dst_stride4); - res8 = __lsx_vilvl_w(reg5, reg1); - res9 = __lsx_vilvh_w(reg5, reg1); - ILVLH_D(res2, res8, res3, res9, dst0, dst1, dst2, dst3); - LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3, - dst_stride4); - res8 = __lsx_vilvl_w(reg6, reg2); - res9 = __lsx_vilvh_w(reg6, reg2); - ILVLH_D(res4, res8, res5, res9, dst0, dst1, dst2, dst3); - LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3, - dst_stride4); - res8 = __lsx_vilvl_w(reg7, reg3); - res9 = __lsx_vilvh_w(reg7, reg3); - ILVLH_D(res6, res8, res7, res9, dst0, dst1, dst2, dst3); - LSX_ST_4(dst0, dst1, dst2, dst3, dst, dst_stride, dst_stride2, dst_stride3, - dst_stride4); - src += 16; - } + "vstelm.d $vr8, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr8, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr9, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr9, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr10, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr10, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr11, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr11, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr12, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr12, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr13, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr13, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr14, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr14, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr15, %[dst], 0, 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vstelm.d $vr15, %[dst], 0, 1 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15"); } -void TransposeUVWx16_LSX(const uint8_t* src, - int src_stride, - uint8_t* dst_a, - int dst_stride_a, - uint8_t* dst_b, - int dst_stride_b, - int width) { - int x; - int len = width / 8; - uint8_t* s; - int src_stride2 = src_stride << 1; - int src_stride3 = src_stride + src_stride2; - int src_stride4 = src_stride2 << 1; - int dst_stride_a2 = dst_stride_a << 1; - int dst_stride_b2 = dst_stride_b << 1; - __m128i src0, src1, src2, src3, dst0, dst1, dst2, dst3; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; - __m128i res0, res1, res2, res3, res4, res5, res6, res7, res8, res9; +void TransposeUVWx8_LSX(const uint8_t* src, + int src_stride, + uint8_t* dst_a, + int dst_stride_a, + uint8_t* dst_b, + int dst_stride_b, + int width) { + const ptrdiff_t s = src_stride; + const ptrdiff_t da = dst_stride_a; + const ptrdiff_t db = dst_stride_b; + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" - for (x = 0; x < len; x++) { - s = (uint8_t*)src; - src0 = __lsx_vld(s, 0); - src1 = __lsx_vldx(s, src_stride); - src2 = __lsx_vldx(s, src_stride2); - src3 = __lsx_vldx(s, src_stride3); - s += src_stride4; - ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); - ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3); - src0 = __lsx_vld(s, 0); - src1 = __lsx_vldx(s, src_stride); - src2 = __lsx_vldx(s, src_stride2); - src3 = __lsx_vldx(s, src_stride3); - s += src_stride4; - ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); - ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7); - ILVLH_W(reg0, reg4, reg1, reg5, res0, res1, res2, res3); - ILVLH_W(reg2, reg6, reg3, reg7, res4, res5, res6, res7); - src0 = __lsx_vld(s, 0); - src1 = __lsx_vldx(s, src_stride); - src2 = __lsx_vldx(s, src_stride2); - src3 = __lsx_vldx(s, src_stride3); - s += src_stride4; - ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); - ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg0, reg1, reg2, reg3); - src0 = __lsx_vld(s, 0); - src1 = __lsx_vldx(s, src_stride); - src2 = __lsx_vldx(s, src_stride2); - src3 = __lsx_vldx(s, src_stride3); - s += src_stride4; - ILVLH_B(src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3); - ILVLH_H(tmp0, tmp2, tmp1, tmp3, reg4, reg5, reg6, reg7); - res8 = __lsx_vilvl_w(reg4, reg0); - res9 = __lsx_vilvh_w(reg4, reg0); - ILVLH_D(res0, res8, res1, res9, dst0, dst1, dst2, dst3); - LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2); - LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2); - res8 = __lsx_vilvl_w(reg5, reg1); - res9 = __lsx_vilvh_w(reg5, reg1); - ILVLH_D(res2, res8, res3, res9, dst0, dst1, dst2, dst3); - LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2); - LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2); - res8 = __lsx_vilvl_w(reg6, reg2); - res9 = __lsx_vilvh_w(reg6, reg2); - ILVLH_D(res4, res8, res5, res9, dst0, dst1, dst2, dst3); - LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2); - LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2); - res8 = __lsx_vilvl_w(reg7, reg3); - res9 = __lsx_vilvh_w(reg7, reg3); - ILVLH_D(res6, res8, res7, res9, dst0, dst1, dst2, dst3); - LSX_ST_2(dst0, dst2, dst_a, dst_stride_a, dst_stride_a2); - LSX_ST_2(dst1, dst3, dst_b, dst_stride_b, dst_stride_b2); - src += 16; - } + "vilvl.b $vr8, $vr1, $vr0 \n" + "vilvh.b $vr9, $vr1, $vr0 \n" + "vilvl.b $vr10, $vr3, $vr2 \n" + "vilvh.b $vr11, $vr3, $vr2 \n" + "vilvl.b $vr12, $vr5, $vr4 \n" + "vilvh.b $vr13, $vr5, $vr4 \n" + "vilvl.b $vr14, $vr7, $vr6 \n" + "vilvh.b $vr15, $vr7, $vr6 \n" + + "vilvl.h $vr0, $vr10, $vr8 \n" + "vilvh.h $vr1, $vr10, $vr8 \n" + "vilvl.h $vr2, $vr11, $vr9 \n" + "vilvh.h $vr3, $vr11, $vr9 \n" + "vilvl.h $vr4, $vr14, $vr12 \n" + "vilvh.h $vr5, $vr14, $vr12 \n" + "vilvl.h $vr6, $vr15, $vr13 \n" + "vilvh.h $vr7, $vr15, $vr13 \n" + + "vilvl.w $vr8, $vr4, $vr0 \n" + "vilvh.w $vr9, $vr4, $vr0 \n" + "vilvl.w $vr10, $vr5, $vr1 \n" + "vilvh.w $vr11, $vr5, $vr1 \n" + "vilvl.w $vr12, $vr6, $vr2 \n" + "vilvh.w $vr13, $vr6, $vr2 \n" + "vilvl.w $vr14, $vr7, $vr3 \n" + "vilvh.w $vr15, $vr7, $vr3 \n" + + "vstelm.d $vr8, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr8, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr9, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr9, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr10, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr10, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr11, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr11, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr12, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr12, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr13, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr13, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr14, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr14, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "vstelm.d $vr15, %[dst_a], 0, 0 \n" + "add.d %[dst_a], %[dst_a], %[da] \n" + "vstelm.d $vr15, %[dst_b], 0, 1 \n" + "add.d %[dst_b], %[dst_b], %[db] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst_a] "+r"(dst_a), // %[dst_a] + [dst_b] "+r"(dst_b), // %[dst_b] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [da] "r"(da), // %[da] + [db] "r"(db) // %[db] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15"); } void TransposeWx4_32_LSX(const uint8_t* src, @@ -217,29 +233,57 @@ void TransposeWx4_32_LSX(const uint8_t* src, uint8_t* dst, int dst_stride, int width) { - int x; - int len = width / 4; - ptrdiff_t src_stride2 = src_stride * 2; - ptrdiff_t src_stride3 = src_stride + src_stride2; - ptrdiff_t src_stride4 = src_stride * 4; - ptrdiff_t dst_stride2 = dst_stride * 2; - ptrdiff_t dst_stride3 = dst_stride + dst_stride2; - __m128i src0, src1, src2, src3; - __m128i dst0, dst1, dst2, dst3; + const ptrdiff_t s = src_stride; + const ptrdiff_t s2 = s * 2; + const ptrdiff_t s3 = s + s2; + const ptrdiff_t s4 = s * 4; + uint8_t* dst0 = dst; + uint8_t* dst1 = dst0 + dst_stride; + uint8_t* dst2 = dst1 + dst_stride; + uint8_t* dst3 = dst2 + dst_stride; + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vldx $vr1, %[src], %[s] \n" + "vldx $vr2, %[src], %[s2] \n" + "vldx $vr3, %[src], %[s3] \n" + "add.d %[src], %[src], %[s4] \n" - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src, 0); - src1 = __lsx_vldx(src, src_stride); - src2 = __lsx_vldx(src, src_stride2); - src3 = __lsx_vldx(src, src_stride3); - LSX_TRANSPOSE4x4_W(src0, src1, src2, src3, dst0, dst1, dst2, dst3); - __lsx_vst(dst0, dst, 0); - __lsx_vstx(dst1, dst, dst_stride); - __lsx_vstx(dst2, dst, dst_stride2); - __lsx_vstx(dst3, dst, dst_stride3); - dst += 16; - src += src_stride4; - } + "vilvl.w $vr4, $vr1, $vr0 \n" + "vilvh.w $vr5, $vr1, $vr0 \n" + "vilvl.w $vr6, $vr3, $vr2 \n" + "vilvh.w $vr7, $vr3, $vr2 \n" + + "vilvl.d $vr0, $vr6, $vr4 \n" + "vilvh.d $vr1, $vr6, $vr4 \n" + "vilvl.d $vr2, $vr7, $vr5 \n" + "vilvh.d $vr3, $vr7, $vr5 \n" + + "vst $vr0, %[dst0], 0 \n" + "vst $vr1, %[dst1], 0 \n" + "vst $vr2, %[dst2], 0 \n" + "vst $vr3, %[dst3], 0 \n" + "addi.d %[dst0], %[dst0], 16 \n" + "addi.d %[dst1], %[dst1], 16 \n" + "addi.d %[dst2], %[dst2], 16 \n" + "addi.d %[dst3], %[dst3], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src), // %[src] + [dst0] "+r"(dst0), // %[dst0] + [dst1] "+r"(dst1), // %[dst1] + [dst2] "+r"(dst2), // %[dst2] + [dst3] "+r"(dst3), // %[dst3] + [w] "+r"(width) // %[w] + : [s] "r"(s), // %[s] + [s2] "r"(s2), // %[s2] + [s3] "r"(s3), // %[s3] + [s4] "r"(s4) // %[s4] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); } void TransposeWx8_16_LSX(const uint16_t* src, @@ -251,46 +295,84 @@ void TransposeWx8_16_LSX(const uint16_t* src, const ptrdiff_t d = (ptrdiff_t)dst_stride * 2; const uint8_t* src_b = (const uint8_t*)src; uint8_t* dst_b = (uint8_t*)dst; - while (width > 0) { - __m128i r0 = __lsx_vld(src_b, 0); - __m128i r1 = __lsx_vldx(src_b, s); - __m128i r2 = __lsx_vldx(src_b, s * 2); - __m128i r3 = __lsx_vldx(src_b, s * 3); - __m128i r4 = __lsx_vldx(src_b, s * 4); - __m128i r5 = __lsx_vldx(src_b, s * 5); - __m128i r6 = __lsx_vldx(src_b, s * 6); - __m128i r7 = __lsx_vldx(src_b, s * 7); + const uint8_t* t; + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "move %[t], %[src] \n" + "vld $vr0, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr1, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr2, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr3, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr4, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr5, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr6, %[t], 0 \n" + "add.d %[t], %[t], %[s] \n" + "vld $vr7, %[t], 0 \n" + "addi.d %[src], %[src], 16 \n" - __m128i t0 = __lsx_vilvl_h(r1, r0); - __m128i t1 = __lsx_vilvh_h(r1, r0); - __m128i t2 = __lsx_vilvl_h(r3, r2); - __m128i t3 = __lsx_vilvh_h(r3, r2); - __m128i t4 = __lsx_vilvl_h(r5, r4); - __m128i t5 = __lsx_vilvh_h(r5, r4); - __m128i t6 = __lsx_vilvl_h(r7, r6); - __m128i t7 = __lsx_vilvh_h(r7, r6); + "vilvl.h $vr8, $vr1, $vr0 \n" + "vilvh.h $vr9, $vr1, $vr0 \n" + "vilvl.h $vr10, $vr3, $vr2 \n" + "vilvh.h $vr11, $vr3, $vr2 \n" + "vilvl.h $vr12, $vr5, $vr4 \n" + "vilvh.h $vr13, $vr5, $vr4 \n" + "vilvl.h $vr14, $vr7, $vr6 \n" + "vilvh.h $vr15, $vr7, $vr6 \n" - __m128i u0 = __lsx_vilvl_w(t2, t0); - __m128i u1 = __lsx_vilvh_w(t2, t0); - __m128i u2 = __lsx_vilvl_w(t3, t1); - __m128i u3 = __lsx_vilvh_w(t3, t1); - __m128i u4 = __lsx_vilvl_w(t6, t4); - __m128i u5 = __lsx_vilvh_w(t6, t4); - __m128i u6 = __lsx_vilvl_w(t7, t5); - __m128i u7 = __lsx_vilvh_w(t7, t5); + "vilvl.w $vr0, $vr10, $vr8 \n" + "vilvh.w $vr1, $vr10, $vr8 \n" + "vilvl.w $vr2, $vr11, $vr9 \n" + "vilvh.w $vr3, $vr11, $vr9 \n" + "vilvl.w $vr4, $vr14, $vr12 \n" + "vilvh.w $vr5, $vr14, $vr12 \n" + "vilvl.w $vr6, $vr15, $vr13 \n" + "vilvh.w $vr7, $vr15, $vr13 \n" - __lsx_vst(__lsx_vilvl_d(u4, u0), dst_b, 0); - __lsx_vstx(__lsx_vilvh_d(u4, u0), dst_b, d); - __lsx_vstx(__lsx_vilvl_d(u5, u1), dst_b, d * 2); - __lsx_vstx(__lsx_vilvh_d(u5, u1), dst_b, d * 3); - __lsx_vstx(__lsx_vilvl_d(u6, u2), dst_b, d * 4); - __lsx_vstx(__lsx_vilvh_d(u6, u2), dst_b, d * 5); - __lsx_vstx(__lsx_vilvl_d(u7, u3), dst_b, d * 6); - __lsx_vstx(__lsx_vilvh_d(u7, u3), dst_b, d * 7); - src_b += 16; - dst_b += d * 8; - width -= 8; - } + "vilvl.d $vr8, $vr4, $vr0 \n" + "vilvh.d $vr9, $vr4, $vr0 \n" + "vilvl.d $vr10, $vr5, $vr1 \n" + "vilvh.d $vr11, $vr5, $vr1 \n" + "vilvl.d $vr12, $vr6, $vr2 \n" + "vilvh.d $vr13, $vr6, $vr2 \n" + "vilvl.d $vr14, $vr7, $vr3 \n" + "vilvh.d $vr15, $vr7, $vr3 \n" + + "vst $vr8, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr9, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr10, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr11, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr12, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr13, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr14, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "vst $vr15, %[dst], 0 \n" + "add.d %[dst], %[dst], %[d] \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_b), // %[src] + [dst] "+r"(dst_b), // %[dst] + [w] "+r"(width), // %[w] + [t] "=&r"(t) // %[t] + : [s] "r"(s), // %[s] + [d] "r"(d) // %[d] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15"); } #ifdef __cplusplus diff --git a/source/row_any.cc b/source/row_any.cc index dbd724915..e1805ab3b 100644 --- a/source/row_any.cc +++ b/source/row_any.cc @@ -115,7 +115,7 @@ ANY41C(I444AlphaToARGBRow_Any_NEON, I444AlphaToARGBRow_NEON, 0, 0, 4, 7) ANY41C(I422AlphaToARGBRow_Any_NEON, I422AlphaToARGBRow_NEON, 1, 0, 4, 7) #endif #ifdef HAS_I422ALPHATOARGBROW_LSX -ANY41C(I422AlphaToARGBRow_Any_LSX, I422AlphaToARGBRow_LSX, 1, 0, 4, 15) +ANY41C(I422AlphaToARGBRow_Any_LSX, I422AlphaToARGBRow_LSX, 1, 0, 4, 7) #endif #ifdef HAS_I422ALPHATOARGBROW_LASX ANY41C(I422AlphaToARGBRow_Any_LASX, I422AlphaToARGBRow_LASX, 1, 0, 4, 15) @@ -452,23 +452,26 @@ ANY31C(I422ToARGB1555Row_Any_NEON, I422ToARGB1555Row_NEON, 1, 0, 2, 7) ANY31C(I422ToRGB565Row_Any_NEON, I422ToRGB565Row_NEON, 1, 0, 2, 7) #endif #ifdef HAS_I422TOARGBROW_LSX -ANY31C(I422ToARGBRow_Any_LSX, I422ToARGBRow_LSX, 1, 0, 4, 15) -ANY31C(I422ToRGBARow_Any_LSX, I422ToRGBARow_LSX, 1, 0, 4, 15) -ANY31C(I422ToRGB24Row_Any_LSX, I422ToRGB24Row_LSX, 1, 0, 3, 15) -ANY31C(I422ToRGB565Row_Any_LSX, I422ToRGB565Row_LSX, 1, 0, 2, 15) -ANY31C(I422ToARGB4444Row_Any_LSX, I422ToARGB4444Row_LSX, 1, 0, 2, 15) -ANY31C(I422ToARGB1555Row_Any_LSX, I422ToARGB1555Row_LSX, 1, 0, 2, 15) +ANY31C(I422ToARGBRow_Any_LSX, I422ToARGBRow_LSX, 1, 0, 4, 7) +ANY31C(I422ToRGBARow_Any_LSX, I422ToRGBARow_LSX, 1, 0, 4, 7) +ANY31C(I422ToRGB24Row_Any_LSX, I422ToRGB24Row_LSX, 1, 0, 3, 7) +ANY31C(I422ToRGB565Row_Any_LSX, I422ToRGB565Row_LSX, 1, 0, 2, 7) +ANY31C(I422ToARGB4444Row_Any_LSX, I422ToARGB4444Row_LSX, 1, 0, 2, 7) +ANY31C(I422ToARGB1555Row_Any_LSX, I422ToARGB1555Row_LSX, 1, 0, 2, 7) #endif #ifdef HAS_I422TOARGBROW_LASX -ANY31C(I422ToARGBRow_Any_LASX, I422ToARGBRow_LASX, 1, 0, 4, 31) -ANY31C(I422ToRGBARow_Any_LASX, I422ToRGBARow_LASX, 1, 0, 4, 31) -ANY31C(I422ToRGB24Row_Any_LASX, I422ToRGB24Row_LASX, 1, 0, 3, 31) -ANY31C(I422ToRGB565Row_Any_LASX, I422ToRGB565Row_LASX, 1, 0, 2, 31) -ANY31C(I422ToARGB4444Row_Any_LASX, I422ToARGB4444Row_LASX, 1, 0, 2, 31) -ANY31C(I422ToARGB1555Row_Any_LASX, I422ToARGB1555Row_LASX, 1, 0, 2, 31) +ANY31C(I422ToARGBRow_Any_LASX, I422ToARGBRow_LASX, 1, 0, 4, 15) +ANY31C(I422ToRGBARow_Any_LASX, I422ToRGBARow_LASX, 1, 0, 4, 15) +ANY31C(I422ToRGB24Row_Any_LASX, I422ToRGB24Row_LASX, 1, 0, 3, 15) +ANY31C(I422ToRGB565Row_Any_LASX, I422ToRGB565Row_LASX, 1, 0, 2, 15) +ANY31C(I422ToARGB4444Row_Any_LASX, I422ToARGB4444Row_LASX, 1, 0, 2, 15) +ANY31C(I422ToARGB1555Row_Any_LASX, I422ToARGB1555Row_LASX, 1, 0, 2, 15) #endif #ifdef HAS_I444TOARGBROW_LSX -ANY31C(I444ToARGBRow_Any_LSX, I444ToARGBRow_LSX, 0, 0, 4, 15) +ANY31C(I444ToARGBRow_Any_LSX, I444ToARGBRow_LSX, 0, 0, 4, 7) +#endif +#ifdef HAS_I444TOARGBROW_LASX +ANY31C(I444ToARGBRow_Any_LASX, I444ToARGBRow_LASX, 0, 0, 4, 15) #endif #ifdef HAS_I422TOARGBROW_WASMSIMD ANY31C(I422ToARGBRow_Any_WASMSIMD, I422ToARGBRow_WASMSIMD, 1, 0, 4, 7) @@ -684,6 +687,9 @@ ANY21(MergeUVRow_Any_NEON, MergeUVRow_NEON, 0, 1, 1, 2, 15) #ifdef HAS_MERGEUVROW_LSX ANY21(MergeUVRow_Any_LSX, MergeUVRow_LSX, 0, 1, 1, 2, 15) #endif +#ifdef HAS_MERGEUVROW_LASX +ANY21(MergeUVRow_Any_LASX, MergeUVRow_LASX, 0, 1, 1, 2, 31) +#endif #ifdef HAS_NV21TOYUV24ROW_NEON ANY21(NV21ToYUV24Row_Any_NEON, NV21ToYUV24Row_NEON, 1, 1, 2, 3, 15) #endif @@ -722,7 +728,10 @@ ANY21(ARGBAddRow_Any_NEON, ARGBAddRow_NEON, 0, 4, 4, 4, 7) ANY21(ARGBSubtractRow_Any_NEON, ARGBSubtractRow_NEON, 0, 4, 4, 4, 7) #endif #ifdef HAS_ARGBBLENDROW_LSX -ANY21(ARGBBlendRow_Any_LSX, ARGBBlendRow_LSX, 0, 4, 4, 4, 7) +ANY21(ARGBBlendRow_Any_LSX, ARGBBlendRow_LSX, 0, 4, 4, 4, 3) +#endif +#ifdef HAS_ARGBBLENDROW_LASX +ANY21(ARGBBlendRow_Any_LASX, ARGBBlendRow_LASX, 0, 4, 4, 4, 7) #endif #ifdef HAS_ARGBMULTIPLYROW_LSX ANY21(ARGBMultiplyRow_Any_LSX, ARGBMultiplyRow_LSX, 0, 4, 4, 4, 3) @@ -749,7 +758,10 @@ ANY21(SobelRow_Any_SSE2, SobelRow_SSE2, 0, 1, 1, 4, 15) ANY21(SobelRow_Any_NEON, SobelRow_NEON, 0, 1, 1, 4, 7) #endif #ifdef HAS_SOBELROW_LSX -ANY21(SobelRow_Any_LSX, SobelRow_LSX, 0, 1, 1, 4, 15) +ANY21(SobelRow_Any_LSX, SobelRow_LSX, 0, 1, 1, 4, 7) +#endif +#ifdef HAS_SOBELROW_LASX +ANY21(SobelRow_Any_LASX, SobelRow_LASX, 0, 1, 1, 4, 15) #endif #ifdef HAS_SOBELTOPLANEROW_SSE2 ANY21(SobelToPlaneRow_Any_SSE2, SobelToPlaneRow_SSE2, 0, 1, 1, 1, 15) @@ -758,7 +770,10 @@ ANY21(SobelToPlaneRow_Any_SSE2, SobelToPlaneRow_SSE2, 0, 1, 1, 1, 15) ANY21(SobelToPlaneRow_Any_NEON, SobelToPlaneRow_NEON, 0, 1, 1, 1, 15) #endif #ifdef HAS_SOBELTOPLANEROW_LSX -ANY21(SobelToPlaneRow_Any_LSX, SobelToPlaneRow_LSX, 0, 1, 1, 1, 31) +ANY21(SobelToPlaneRow_Any_LSX, SobelToPlaneRow_LSX, 0, 1, 1, 1, 15) +#endif +#ifdef HAS_SOBELTOPLANEROW_LASX +ANY21(SobelToPlaneRow_Any_LASX, SobelToPlaneRow_LASX, 0, 1, 1, 1, 31) #endif #ifdef HAS_SOBELXYROW_SSE2 ANY21(SobelXYRow_Any_SSE2, SobelXYRow_SSE2, 0, 1, 1, 4, 15) @@ -767,7 +782,10 @@ ANY21(SobelXYRow_Any_SSE2, SobelXYRow_SSE2, 0, 1, 1, 4, 15) ANY21(SobelXYRow_Any_NEON, SobelXYRow_NEON, 0, 1, 1, 4, 7) #endif #ifdef HAS_SOBELXYROW_LSX -ANY21(SobelXYRow_Any_LSX, SobelXYRow_LSX, 0, 1, 1, 4, 15) +ANY21(SobelXYRow_Any_LSX, SobelXYRow_LSX, 0, 1, 1, 4, 7) +#endif +#ifdef HAS_SOBELXYROW_LASX +ANY21(SobelXYRow_Any_LASX, SobelXYRow_LASX, 0, 1, 1, 4, 15) #endif #ifdef HAS_MERGEUVROW_WASMSIMD ANY21(MergeUVRow_Any_WASMSIMD, MergeUVRow_WASMSIMD, 0, 1, 1, 2, 15) @@ -1035,10 +1053,10 @@ ANY11(CopyRow_Any_SSE2, CopyRow_SSE2, 0, 1, 1, 31) ANY11(CopyRow_Any_NEON, CopyRow_NEON, 0, 1, 1, 31) #endif #ifdef HAS_COPYROW_LSX -ANY11(CopyRow_Any_LSX, CopyRow_LSX, 0, 1, 1, 31) +ANY11(CopyRow_Any_LSX, CopyRow_LSX, 0, 1, 1, 15) #endif #ifdef HAS_COPYROW_LASX -ANY11(CopyRow_Any_LASX, CopyRow_LASX, 0, 1, 1, 63) +ANY11(CopyRow_Any_LASX, CopyRow_LASX, 0, 1, 1, 31) #endif #if defined(HAS_ARGBTORGB24ROW_SSSE3) ANY11(ARGBToRGB24Row_Any_SSSE3, ARGBToRGB24Row_SSSE3, 0, 4, 3, 15) @@ -1143,7 +1161,10 @@ ANY11(ARGBToARGB1555Row_Any_LASX, ARGBToARGB1555Row_LASX, 0, 4, 2, 15) ANY11(ARGBToARGB4444Row_Any_LASX, ARGBToARGB4444Row_LASX, 0, 4, 2, 15) #endif #if defined(HAS_J400TOARGBROW_LSX) -ANY11(J400ToARGBRow_Any_LSX, J400ToARGBRow_LSX, 0, 1, 4, 15) +ANY11(J400ToARGBRow_Any_LSX, J400ToARGBRow_LSX, 0, 1, 4, 7) +#endif +#if defined(HAS_J400TOARGBROW_LASX) +ANY11(J400ToARGBRow_Any_LASX, J400ToARGBRow_LASX, 0, 1, 4, 15) #endif #if defined(HAS_RAWTORGB24ROW_NEON) ANY11(RAWToRGB24Row_Any_NEON, RAWToRGB24Row_NEON, 0, 3, 3, 15) @@ -1173,10 +1194,10 @@ ANY11(UYVYToYRow_Any_SSE2, UYVYToYRow_SSE2, 1, 4, 1, 15) ANY11(RGB565ToYRow_Any_NEON, RGB565ToYRow_NEON, 0, 2, 1, 15) #endif #ifdef HAS_RGB565TOYROW_LSX -ANY11(RGB565ToYRow_Any_LSX, RGB565ToYRow_LSX, 0, 2, 1, 15) +ANY11(RGB565ToYRow_Any_LSX, RGB565ToYRow_LSX, 0, 2, 1, 7) #endif #ifdef HAS_RGB565TOYROW_LASX -ANY11(RGB565ToYRow_Any_LASX, RGB565ToYRow_LASX, 0, 2, 1, 31) +ANY11(RGB565ToYRow_Any_LASX, RGB565ToYRow_LASX, 0, 2, 1, 15) #endif #ifdef HAS_ARGB1555TOYROW_NEON #ifdef __aarch64__ @@ -1186,10 +1207,10 @@ ANY11(ARGB1555ToYRow_Any_NEON, ARGB1555ToYRow_NEON, 0, 2, 1, 7) #endif #endif #ifdef HAS_ARGB1555TOYROW_LSX -ANY11(ARGB1555ToYRow_Any_LSX, ARGB1555ToYRow_LSX, 0, 2, 1, 15) +ANY11(ARGB1555ToYRow_Any_LSX, ARGB1555ToYRow_LSX, 0, 2, 1, 7) #endif #ifdef HAS_ARGB1555TOYROW_LASX -ANY11(ARGB1555ToYRow_Any_LASX, ARGB1555ToYRow_LASX, 0, 2, 1, 31) +ANY11(ARGB1555ToYRow_Any_LASX, ARGB1555ToYRow_LASX, 0, 2, 1, 15) #endif #ifdef HAS_ARGB4444TOYROW_NEON #ifdef __aarch64__ @@ -1253,28 +1274,28 @@ ANY11(RAWToARGBRow_Any_LASX, RAWToARGBRow_LASX, 0, 3, 4, 31) ANY11(RGB565ToARGBRow_Any_NEON, RGB565ToARGBRow_NEON, 0, 2, 4, 15) #endif #ifdef HAS_RGB565TOARGBROW_LSX -ANY11(RGB565ToARGBRow_Any_LSX, RGB565ToARGBRow_LSX, 0, 2, 4, 15) +ANY11(RGB565ToARGBRow_Any_LSX, RGB565ToARGBRow_LSX, 0, 2, 4, 7) #endif #ifdef HAS_RGB565TOARGBROW_LASX -ANY11(RGB565ToARGBRow_Any_LASX, RGB565ToARGBRow_LASX, 0, 2, 4, 31) +ANY11(RGB565ToARGBRow_Any_LASX, RGB565ToARGBRow_LASX, 0, 2, 4, 15) #endif #ifdef HAS_ARGB1555TOARGBROW_NEON ANY11(ARGB1555ToARGBRow_Any_NEON, ARGB1555ToARGBRow_NEON, 0, 2, 4, 15) #endif #ifdef HAS_ARGB1555TOARGBROW_LSX -ANY11(ARGB1555ToARGBRow_Any_LSX, ARGB1555ToARGBRow_LSX, 0, 2, 4, 15) +ANY11(ARGB1555ToARGBRow_Any_LSX, ARGB1555ToARGBRow_LSX, 0, 2, 4, 7) #endif #ifdef HAS_ARGB1555TOARGBROW_LASX -ANY11(ARGB1555ToARGBRow_Any_LASX, ARGB1555ToARGBRow_LASX, 0, 2, 4, 31) +ANY11(ARGB1555ToARGBRow_Any_LASX, ARGB1555ToARGBRow_LASX, 0, 2, 4, 15) #endif #ifdef HAS_ARGB4444TOARGBROW_NEON ANY11(ARGB4444ToARGBRow_Any_NEON, ARGB4444ToARGBRow_NEON, 0, 2, 4, 7) #endif #ifdef HAS_ARGB4444TOARGBROW_LSX -ANY11(ARGB4444ToARGBRow_Any_LSX, ARGB4444ToARGBRow_LSX, 0, 2, 4, 15) +ANY11(ARGB4444ToARGBRow_Any_LSX, ARGB4444ToARGBRow_LSX, 0, 2, 4, 7) #endif #ifdef HAS_ARGB4444TOARGBROW_LASX -ANY11(ARGB4444ToARGBRow_Any_LASX, ARGB4444ToARGBRow_LASX, 0, 2, 4, 31) +ANY11(ARGB4444ToARGBRow_Any_LASX, ARGB4444ToARGBRow_LASX, 0, 2, 4, 15) #endif #ifdef HAS_ARGBATTENUATEROW_SSSE3 ANY11(ARGBAttenuateRow_Any_SSSE3, ARGBAttenuateRow_SSSE3, 0, 4, 4, 3) @@ -1292,10 +1313,10 @@ ANY11(ARGBUnattenuateRow_Any_AVX2, ARGBUnattenuateRow_AVX2, 0, 4, 4, 7) ANY11(ARGBAttenuateRow_Any_NEON, ARGBAttenuateRow_NEON, 0, 4, 4, 7) #endif #ifdef HAS_ARGBATTENUATEROW_LSX -ANY11(ARGBAttenuateRow_Any_LSX, ARGBAttenuateRow_LSX, 0, 4, 4, 7) +ANY11(ARGBAttenuateRow_Any_LSX, ARGBAttenuateRow_LSX, 0, 4, 4, 3) #endif #ifdef HAS_ARGBATTENUATEROW_LASX -ANY11(ARGBAttenuateRow_Any_LASX, ARGBAttenuateRow_LASX, 0, 4, 4, 15) +ANY11(ARGBAttenuateRow_Any_LASX, ARGBAttenuateRow_LASX, 0, 4, 4, 7) #endif #ifdef HAS_ARGBEXTRACTALPHAROW_SSE2 ANY11(ARGBExtractAlphaRow_Any_SSE2, ARGBExtractAlphaRow_SSE2, 0, 4, 1, 7) @@ -1307,7 +1328,10 @@ ANY11(ARGBExtractAlphaRow_Any_AVX2, ARGBExtractAlphaRow_AVX2, 0, 4, 1, 31) ANY11(ARGBExtractAlphaRow_Any_NEON, ARGBExtractAlphaRow_NEON, 0, 4, 1, 15) #endif #ifdef HAS_ARGBEXTRACTALPHAROW_LSX -ANY11(ARGBExtractAlphaRow_Any_LSX, ARGBExtractAlphaRow_LSX, 0, 4, 1, 15) +ANY11(ARGBExtractAlphaRow_Any_LSX, ARGBExtractAlphaRow_LSX, 0, 4, 1, 7) +#endif +#ifdef HAS_ARGBEXTRACTALPHAROW_LASX +ANY11(ARGBExtractAlphaRow_Any_LASX, ARGBExtractAlphaRow_LASX, 0, 4, 1, 15) #endif #ifdef HAS_ARGBEXTRACTALPHAROW_WASMSIMD ANY11(ARGBExtractAlphaRow_Any_WASMSIMD, @@ -1444,6 +1468,14 @@ ANY11P(I400ToARGBRow_Any_LSX, const struct YuvConstants*, 1, 4, + 7) +#endif +#if defined(HAS_I400TOARGBROW_LASX) +ANY11P(I400ToARGBRow_Any_LASX, + I400ToARGBRow_LASX, + const struct YuvConstants*, + 1, + 4, 15) #endif @@ -1469,7 +1501,7 @@ ANY11P(ARGBToRGB565DitherRow_Any_LSX, const uint32_t, 4, 2, - 7) + 3) #endif #if defined(HAS_ARGBTORGB565DITHERROW_LASX) ANY11P(ARGBToRGB565DitherRow_Any_LASX, @@ -1477,7 +1509,7 @@ ANY11P(ARGBToRGB565DitherRow_Any_LASX, const uint32_t, 4, 2, - 15) + 7) #endif #ifdef HAS_ARGBSHUFFLEROW_SSSE3 ANY11P(ARGBShuffleRow_Any_SSSE3, ARGBShuffleRow_SSSE3, const uint8_t*, 4, 4, 7) @@ -1489,10 +1521,10 @@ ANY11P(ARGBShuffleRow_Any_AVX2, ARGBShuffleRow_AVX2, const uint8_t*, 4, 4, 15) ANY11P(ARGBShuffleRow_Any_NEON, ARGBShuffleRow_NEON, const uint8_t*, 4, 4, 3) #endif #ifdef HAS_ARGBSHUFFLEROW_LSX -ANY11P(ARGBShuffleRow_Any_LSX, ARGBShuffleRow_LSX, const uint8_t*, 4, 4, 7) +ANY11P(ARGBShuffleRow_Any_LSX, ARGBShuffleRow_LSX, const uint8_t*, 4, 4, 3) #endif #ifdef HAS_ARGBSHUFFLEROW_LASX -ANY11P(ARGBShuffleRow_Any_LASX, ARGBShuffleRow_LASX, const uint8_t*, 4, 4, 15) +ANY11P(ARGBShuffleRow_Any_LASX, ARGBShuffleRow_LASX, const uint8_t*, 4, 4, 7) #endif #if defined(HAS_ARGBTORGB565DITHERROW_WASMSIMD) ANY11P(ARGBToRGB565DitherRow_Any_WASMSIMD, @@ -1807,7 +1839,10 @@ ANY11P16(HalfFloatRow_Any_NEON, HalfFloatRow_NEON, uint16_t, uint16_t, 2, 2, 15) ANY11P16(ByteToFloatRow_Any_NEON, ByteToFloatRow_NEON, uint8_t, float, 1, 3, 7) #endif #ifdef HAS_HALFFLOATROW_LSX -ANY11P16(HalfFloatRow_Any_LSX, HalfFloatRow_LSX, uint16_t, uint16_t, 2, 2, 31) +ANY11P16(HalfFloatRow_Any_LSX, HalfFloatRow_LSX, uint16_t, uint16_t, 2, 2, 7) +#endif +#ifdef HAS_HALFFLOATROW_LASX +ANY11P16(HalfFloatRow_Any_LASX, HalfFloatRow_LASX, uint16_t, uint16_t, 2, 2, 15) #endif #undef ANY11P16 @@ -1849,6 +1884,10 @@ ANY11C(UYVYToARGBRow_Any_NEON, UYVYToARGBRow_NEON, 1, 4, 4, 7) ANY11C(YUY2ToARGBRow_Any_LSX, YUY2ToARGBRow_LSX, 1, 4, 4, 7) ANY11C(UYVYToARGBRow_Any_LSX, UYVYToARGBRow_LSX, 1, 4, 4, 7) #endif +#if defined(HAS_YUY2TOARGBROW_LASX) +ANY11C(YUY2ToARGBRow_Any_LASX, YUY2ToARGBRow_LASX, 1, 4, 4, 15) +ANY11C(UYVYToARGBRow_Any_LASX, UYVYToARGBRow_LASX, 1, 4, 4, 15) +#endif #undef ANY11C // Any 1 to 1 interpolate. Takes 2 rows of source via stride. @@ -1885,7 +1924,10 @@ ANY11I(InterpolateRow_Any_AVX2, InterpolateRow_AVX2, uint8_t, uint8_t, 1, 1, 31) ANY11I(InterpolateRow_Any_NEON, InterpolateRow_NEON, uint8_t, uint8_t, 1, 1, 15) #endif #ifdef HAS_INTERPOLATEROW_LSX -ANY11I(InterpolateRow_Any_LSX, InterpolateRow_LSX, uint8_t, uint8_t, 1, 1, 31) +ANY11I(InterpolateRow_Any_LSX, InterpolateRow_LSX, uint8_t, uint8_t, 1, 1, 15) +#endif +#ifdef HAS_INTERPOLATEROW_LASX +ANY11I(InterpolateRow_Any_LASX, InterpolateRow_LASX, uint8_t, uint8_t, 1, 1, 31) #endif #ifdef HAS_INTERPOLATEROW_16_NEON @@ -2057,10 +2099,10 @@ ANY11M(MirrorRow_Any_SSSE3, MirrorRow_SSSE3, 1, 15) ANY11M(MirrorRow_Any_NEON, MirrorRow_NEON, 1, 31) #endif #ifdef HAS_MIRRORROW_LSX -ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 31) +ANY11M(MirrorRow_Any_LSX, MirrorRow_LSX, 1, 15) #endif #ifdef HAS_MIRRORROW_LASX -ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 63) +ANY11M(MirrorRow_Any_LASX, MirrorRow_LASX, 1, 31) #endif #ifdef HAS_MIRRORUVROW_AVX512BW ANY11M(MirrorUVRow_Any_AVX512BW, MirrorUVRow_AVX512BW, 2, 31) @@ -2090,10 +2132,10 @@ ANY11M(ARGBMirrorRow_Any_SSE2, ARGBMirrorRow_SSE2, 4, 3) ANY11M(ARGBMirrorRow_Any_NEON, ARGBMirrorRow_NEON, 4, 7) #endif #ifdef HAS_ARGBMIRRORROW_LSX -ANY11M(ARGBMirrorRow_Any_LSX, ARGBMirrorRow_LSX, 4, 7) +ANY11M(ARGBMirrorRow_Any_LSX, ARGBMirrorRow_LSX, 4, 3) #endif #ifdef HAS_ARGBMIRRORROW_LASX -ANY11M(ARGBMirrorRow_Any_LASX, ARGBMirrorRow_LASX, 4, 15) +ANY11M(ARGBMirrorRow_Any_LASX, ARGBMirrorRow_LASX, 4, 7) #endif #ifdef HAS_RGB24MIRRORROW_AVX2 ANY11M(RGB24MirrorRow_Any_AVX2, RGB24MirrorRow_AVX2, 3, 31) @@ -2140,12 +2182,18 @@ ANY1(SetRow_Any_NEON, SetRow_NEON, uint8_t, 1, 15) #ifdef HAS_SETROW_LSX ANY1(SetRow_Any_LSX, SetRow_LSX, uint8_t, 1, 15) #endif +#ifdef HAS_SETROW_LASX +ANY1(SetRow_Any_LASX, SetRow_LASX, uint8_t, 1, 31) +#endif #ifdef HAS_ARGBSETROW_NEON ANY1(ARGBSetRow_Any_NEON, ARGBSetRow_NEON, uint32_t, 4, 3) #endif #ifdef HAS_ARGBSETROW_LSX ANY1(ARGBSetRow_Any_LSX, ARGBSetRow_LSX, uint32_t, 4, 3) #endif +#ifdef HAS_ARGBSETROW_LASX +ANY1(ARGBSetRow_Any_LASX, ARGBSetRow_LASX, uint32_t, 4, 7) +#endif #undef ANY1 // Any 1 to 2. Outputs UV planes. @@ -2185,7 +2233,10 @@ ANY12(SplitUVRow_Any_AVX2, SplitUVRow_AVX2, 0, 2, 0, 31) ANY12(SplitUVRow_Any_NEON, SplitUVRow_NEON, 0, 2, 0, 15) #endif #ifdef HAS_SPLITUVROW_LSX -ANY12(SplitUVRow_Any_LSX, SplitUVRow_LSX, 0, 2, 0, 31) +ANY12(SplitUVRow_Any_LSX, SplitUVRow_LSX, 0, 2, 0, 15) +#endif +#ifdef HAS_SPLITUVROW_LASX +ANY12(SplitUVRow_Any_LASX, SplitUVRow_LASX, 0, 2, 0, 31) #endif #ifdef HAS_YUY2TOUV422ROW_AVX2 ANY12(YUY2ToUV422Row_Any_AVX2, YUY2ToUV422Row_AVX2, 1, 4, 1, 31) @@ -2567,13 +2618,15 @@ ANY11MC(ARGB1555ToYMatrixRow_Any_NEON, ARGB1555ToYMatrixRow_NEON, 2, 15) ANY11MC(ARGB4444ToYMatrixRow_Any_NEON, ARGB4444ToYMatrixRow_NEON, 2, 15) #endif #ifdef HAS_ARGBTOYMATRIXROW_LSX -ANY11MC(ARGBToYMatrixRow_Any_LSX, ARGBToYMatrixRow_LSX, 4, 15) +ANY11MC(ARGBToYMatrixRow_Any_LSX, ARGBToYMatrixRow_LSX, 4, 7) +#endif +#ifdef HAS_ARGBTOYMATRIXROW_LASX +ANY11MC(ARGBToYMatrixRow_Any_LASX, ARGBToYMatrixRow_LASX, 4, 15) #endif #ifdef HAS_RGBTOYMATRIXROW_LSX ANY11MC(RGBToYMatrixRow_Any_LSX, RGBToYMatrixRow_LSX, 3, 15) #endif -#ifdef HAS_ARGBTOYMATRIXROW_LASX -ANY11MC(ARGBToYMatrixRow_Any_LASX, ARGBToYMatrixRow_LASX, 4, 31) +#ifdef HAS_RGBTOYMATRIXROW_LASX ANY11MC(RGBToYMatrixRow_Any_LASX, RGBToYMatrixRow_LASX, 3, 31) #endif #ifdef HAS_ARGBTOYMATRIXROW_RVV diff --git a/source/row_lasx.cc b/source/row_lasx.cc index 5a43f638a..91d1678ca 100644 --- a/source/row_lasx.cc +++ b/source/row_lasx.cc @@ -1,8 +1,6 @@ /* * Copyright 2022 The LibYuv Project Authors. All rights reserved. * - * Copyright (c) 2022 Loongson Technology Corporation Limited - * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found @@ -12,295 +10,227 @@ #include "libyuv/row.h" -// Note: every register written by an inline asm block (vector, general -// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be -// listed in its clobber list, even when the ABI treats it as caller-saved. -// With LTO/LTCG the compiler may keep values live in any register across the -// asm statement, and other OS ABIs (e.g. Windows) differ in which registers -// are callee-saved. List "memory" first, then "cc", then registers. - #if !defined(LIBYUV_DISABLE_LASX) && defined(__loongarch_asx) -#include "libyuv/loongson_intrinsics.h" + +#include "libyuv/basic_types.h" #ifdef __cplusplus namespace libyuv { extern "C" { #endif -#define ALPHA_VAL (-1) +void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int count) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(count) + : + : "memory", "$xr0"); +} -// Fill YUV -> RGB conversion constants into vectors -#define YUVTORGB_SETUP(yuvconst, ub, vr, ug, vg, yg, yb) \ - { \ - ub = __lasx_xvreplgr2vr_h(yuvconst->kUVToB[0]); \ - vr = __lasx_xvreplgr2vr_h(yuvconst->kUVToR[1]); \ - ug = __lasx_xvreplgr2vr_h(yuvconst->kUVToG[0]); \ - vg = __lasx_xvreplgr2vr_h(yuvconst->kUVToG[1]); \ - yg = __lasx_xvreplgr2vr_h(yuvconst->kYToRgb[0]); \ - yb = __lasx_xvreplgr2vr_w(yuvconst->kYBiasToRgb[0]); \ - } +void SetRow_LASX(uint8_t* dst, uint8_t v8, int count) { + asm volatile( + "xvreplgr2vr.b $xr0, %[val]\n\t" + "1:\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst), [cnt] "+r"(count) + : [val] "r"(v8) + : "memory", "$xr0"); +} -// Load 32 YUV422 pixel data -#define READYUV422_D(psrc_y, psrc_u, psrc_v, out_y, uv_l, uv_h) \ - { \ - __m256i temp0, temp1; \ - \ - DUP2_ARG2(__lasx_xvld, psrc_y, 0, psrc_u, 0, out_y, temp0); \ - temp1 = __lasx_xvld(psrc_v, 0); \ - temp0 = __lasx_xvsub_b(temp0, const_0x80); \ - temp1 = __lasx_xvsub_b(temp1, const_0x80); \ - temp0 = __lasx_vext2xv_h_b(temp0); \ - temp1 = __lasx_vext2xv_h_b(temp1); \ - uv_l = __lasx_xvilvl_h(temp0, temp1); \ - uv_h = __lasx_xvilvh_h(temp0, temp1); \ - } +void ARGBSetRow_LASX(uint8_t* dst_argb, uint32_t v32, int width) { + asm volatile( + "xvreplgr2vr.w $xr0, %[val]\n\t" + "1:\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(v32) + : "memory", "$xr0"); +} -// Load 16 YUV422 pixel data -#define READYUV422(psrc_y, psrc_u, psrc_v, out_y, uv) \ - { \ - __m256i temp0, temp1; \ - \ - out_y = __lasx_xvld(psrc_y, 0); \ - temp0 = __lasx_xvldrepl_d(psrc_u, 0); \ - temp1 = __lasx_xvldrepl_d(psrc_v, 0); \ - uv = __lasx_xvilvl_b(temp0, temp1); \ - uv = __lasx_xvsub_b(uv, const_0x80); \ - uv = __lasx_vext2xv_h_b(uv); \ - } - -// Convert 16 pixels of YUV420 to RGB. -#define YUVTORGB_D(in_y, in_uvl, in_uvh, ubvr, ugvg, yg, yb, b_l, b_h, g_l, \ - g_h, r_l, r_h) \ - { \ - __m256i u_l, u_h, v_l, v_h; \ - __m256i yl_ev, yl_od, yh_ev, yh_od; \ - __m256i temp0, temp1, temp2, temp3; \ - \ - temp0 = __lasx_xvilvl_b(in_y, in_y); \ - temp1 = __lasx_xvilvh_b(in_y, in_y); \ - yl_ev = __lasx_xvmulwev_w_hu_h(temp0, yg); \ - yl_od = __lasx_xvmulwod_w_hu_h(temp0, yg); \ - yh_ev = __lasx_xvmulwev_w_hu_h(temp1, yg); \ - yh_od = __lasx_xvmulwod_w_hu_h(temp1, yg); \ - DUP4_ARG2(__lasx_xvsrai_w, yl_ev, 16, yl_od, 16, yh_ev, 16, yh_od, 16, \ - yl_ev, yl_od, yh_ev, yh_od); \ - yl_ev = __lasx_xvadd_w(yl_ev, yb); \ - yl_od = __lasx_xvadd_w(yl_od, yb); \ - yh_ev = __lasx_xvadd_w(yh_ev, yb); \ - yh_od = __lasx_xvadd_w(yh_od, yb); \ - v_l = __lasx_xvmulwev_w_h(in_uvl, ubvr); \ - u_l = __lasx_xvmulwod_w_h(in_uvl, ubvr); \ - v_h = __lasx_xvmulwev_w_h(in_uvh, ubvr); \ - u_h = __lasx_xvmulwod_w_h(in_uvh, ubvr); \ - temp0 = __lasx_xvadd_w(yl_ev, u_l); \ - temp1 = __lasx_xvadd_w(yl_od, u_l); \ - temp2 = __lasx_xvadd_w(yh_ev, u_h); \ - temp3 = __lasx_xvadd_w(yh_od, u_h); \ - DUP4_ARG2(__lasx_xvsrai_w, temp0, 6, temp1, 6, temp2, 6, temp3, 6, temp0, \ - temp1, temp2, temp3); \ - DUP4_ARG1(__lasx_xvclip255_w, temp0, temp1, temp2, temp3, temp0, temp1, \ - temp2, temp3); \ - b_l = __lasx_xvpackev_h(temp1, temp0); \ - b_h = __lasx_xvpackev_h(temp3, temp2); \ - temp0 = __lasx_xvadd_w(yl_ev, v_l); \ - temp1 = __lasx_xvadd_w(yl_od, v_l); \ - temp2 = __lasx_xvadd_w(yh_ev, v_h); \ - temp3 = __lasx_xvadd_w(yh_od, v_h); \ - DUP4_ARG2(__lasx_xvsrai_w, temp0, 6, temp1, 6, temp2, 6, temp3, 6, temp0, \ - temp1, temp2, temp3); \ - DUP4_ARG1(__lasx_xvclip255_w, temp0, temp1, temp2, temp3, temp0, temp1, \ - temp2, temp3); \ - r_l = __lasx_xvpackev_h(temp1, temp0); \ - r_h = __lasx_xvpackev_h(temp3, temp2); \ - DUP2_ARG2(__lasx_xvdp2_w_h, in_uvl, ugvg, in_uvh, ugvg, u_l, u_h); \ - temp0 = __lasx_xvsub_w(yl_ev, u_l); \ - temp1 = __lasx_xvsub_w(yl_od, u_l); \ - temp2 = __lasx_xvsub_w(yh_ev, u_h); \ - temp3 = __lasx_xvsub_w(yh_od, u_h); \ - DUP4_ARG2(__lasx_xvsrai_w, temp0, 6, temp1, 6, temp2, 6, temp3, 6, temp0, \ - temp1, temp2, temp3); \ - DUP4_ARG1(__lasx_xvclip255_w, temp0, temp1, temp2, temp3, temp0, temp1, \ - temp2, temp3); \ - g_l = __lasx_xvpackev_h(temp1, temp0); \ - g_h = __lasx_xvpackev_h(temp3, temp2); \ - } - -// Convert 8 pixels of YUV420 to RGB. -#define YUVTORGB(in_y, in_uv, ubvr, ugvg, yg, yb, out_b, out_g, out_r) \ - { \ - __m256i u_l, v_l, yl_ev, yl_od; \ - __m256i temp0, temp1; \ - \ - in_y = __lasx_xvpermi_d(in_y, 0xD8); \ - temp0 = __lasx_xvilvl_b(in_y, in_y); \ - yl_ev = __lasx_xvmulwev_w_hu_h(temp0, yg); \ - yl_od = __lasx_xvmulwod_w_hu_h(temp0, yg); \ - DUP2_ARG2(__lasx_xvsrai_w, yl_ev, 16, yl_od, 16, yl_ev, yl_od); \ - yl_ev = __lasx_xvadd_w(yl_ev, yb); \ - yl_od = __lasx_xvadd_w(yl_od, yb); \ - v_l = __lasx_xvmulwev_w_h(in_uv, ubvr); \ - u_l = __lasx_xvmulwod_w_h(in_uv, ubvr); \ - temp0 = __lasx_xvadd_w(yl_ev, u_l); \ - temp1 = __lasx_xvadd_w(yl_od, u_l); \ - DUP2_ARG2(__lasx_xvsrai_w, temp0, 6, temp1, 6, temp0, temp1); \ - DUP2_ARG1(__lasx_xvclip255_w, temp0, temp1, temp0, temp1); \ - out_b = __lasx_xvpackev_h(temp1, temp0); \ - temp0 = __lasx_xvadd_w(yl_ev, v_l); \ - temp1 = __lasx_xvadd_w(yl_od, v_l); \ - DUP2_ARG2(__lasx_xvsrai_w, temp0, 6, temp1, 6, temp0, temp1); \ - DUP2_ARG1(__lasx_xvclip255_w, temp0, temp1, temp0, temp1); \ - out_r = __lasx_xvpackev_h(temp1, temp0); \ - u_l = __lasx_xvdp2_w_h(in_uv, ugvg); \ - temp0 = __lasx_xvsub_w(yl_ev, u_l); \ - temp1 = __lasx_xvsub_w(yl_od, u_l); \ - DUP2_ARG2(__lasx_xvsrai_w, temp0, 6, temp1, 6, temp0, temp1); \ - DUP2_ARG1(__lasx_xvclip255_w, temp0, temp1, temp0, temp1); \ - out_g = __lasx_xvpackev_h(temp1, temp0); \ - } - -// Pack and Store 16 ARGB values. -#define STOREARGB_D(a_l, a_h, r_l, r_h, g_l, g_h, b_l, b_h, pdst_argb) \ - { \ - __m256i temp0, temp1, temp2, temp3; \ - \ - temp0 = __lasx_xvpackev_b(g_l, b_l); \ - temp1 = __lasx_xvpackev_b(a_l, r_l); \ - temp2 = __lasx_xvpackev_b(g_h, b_h); \ - temp3 = __lasx_xvpackev_b(a_h, r_h); \ - r_l = __lasx_xvilvl_h(temp1, temp0); \ - r_h = __lasx_xvilvh_h(temp1, temp0); \ - g_l = __lasx_xvilvl_h(temp3, temp2); \ - g_h = __lasx_xvilvh_h(temp3, temp2); \ - temp0 = __lasx_xvpermi_q(r_h, r_l, 0x20); \ - temp1 = __lasx_xvpermi_q(g_h, g_l, 0x20); \ - temp2 = __lasx_xvpermi_q(r_h, r_l, 0x31); \ - temp3 = __lasx_xvpermi_q(g_h, g_l, 0x31); \ - __lasx_xvst(temp0, pdst_argb, 0); \ - __lasx_xvst(temp1, pdst_argb, 32); \ - __lasx_xvst(temp2, pdst_argb, 64); \ - __lasx_xvst(temp3, pdst_argb, 96); \ - pdst_argb += 128; \ - } - -// Pack and Store 8 ARGB values. -#define STOREARGB(in_a, in_r, in_g, in_b, pdst_argb) \ - { \ - __m256i temp0, temp1, temp2, temp3; \ - \ - temp0 = __lasx_xvpackev_b(in_g, in_b); \ - temp1 = __lasx_xvpackev_b(in_a, in_r); \ - temp2 = __lasx_xvilvl_h(temp1, temp0); \ - temp3 = __lasx_xvilvh_h(temp1, temp0); \ - temp0 = __lasx_xvpermi_q(temp3, temp2, 0x20); \ - temp1 = __lasx_xvpermi_q(temp3, temp2, 0x31); \ - __lasx_xvst(temp0, pdst_argb, 0); \ - __lasx_xvst(temp1, pdst_argb, 32); \ - pdst_argb += 64; \ - } - -#define RGBTOUV(_tmpb, _tmpg, _tmpr, _nexb, _nexg, _nexr, _reg0, _reg1) \ - { \ - __m256i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5; \ - _tmp0 = __lasx_xvaddwev_h_bu(_tmpb, _nexb); \ - _tmp1 = __lasx_xvaddwod_h_bu(_tmpb, _nexb); \ - _tmp2 = __lasx_xvaddwev_h_bu(_tmpg, _nexg); \ - _tmp3 = __lasx_xvaddwod_h_bu(_tmpg, _nexg); \ - _reg0 = __lasx_xvaddwev_h_bu(_tmpr, _nexr); \ - _reg1 = __lasx_xvaddwod_h_bu(_tmpr, _nexr); \ - _tmp4 = __lasx_xvaddwev_w_hu(_tmp0, _tmp1); \ - _tmp5 = __lasx_xvaddwod_w_hu(_tmp0, _tmp1); \ - _tmp0 = __lasx_xvilvl_w(_tmp5, _tmp4); \ - _tmp1 = __lasx_xvilvh_w(_tmp5, _tmp4); \ - _tmpb = __lasx_xvssrarni_hu_w(_tmp1, _tmp0, 2); \ - _tmp4 = __lasx_xvaddwev_w_hu(_tmp2, _tmp3); \ - _tmp5 = __lasx_xvaddwod_w_hu(_tmp2, _tmp3); \ - _tmp2 = __lasx_xvilvl_w(_tmp5, _tmp4); \ - _tmp3 = __lasx_xvilvh_w(_tmp5, _tmp4); \ - _tmpg = __lasx_xvssrarni_hu_w(_tmp3, _tmp2, 2); \ - _tmp4 = __lasx_xvaddwev_w_hu(_reg0, _reg1); \ - _tmp5 = __lasx_xvaddwod_w_hu(_reg0, _reg1); \ - _tmp0 = __lasx_xvilvl_w(_tmp5, _tmp4); \ - _tmp1 = __lasx_xvilvh_w(_tmp5, _tmp4); \ - _tmpr = __lasx_xvssrarni_hu_w(_tmp1, _tmp0, 2); \ - _reg0 = __lasx_xvmadd_h(const_8000, const_112, _tmpb); \ - _reg1 = __lasx_xvmadd_h(const_8000, const_112, _tmpr); \ - _reg0 = __lasx_xvmsub_h(_reg0, const_74, _tmpg); \ - _reg1 = __lasx_xvmsub_h(_reg1, const_94, _tmpg); \ - _reg0 = __lasx_xvmsub_h(_reg0, const_38, _tmpr); \ - _reg1 = __lasx_xvmsub_h(_reg1, const_18, _tmpb); \ - } - -void MirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width) { - int x; - int len = width / 64; - __m256i src0, src1; - __m256i shuffler = {0x08090A0B0C0D0E0F, 0x0001020304050607, - 0x08090A0B0C0D0E0F, 0x0001020304050607}; - src += width - 64; - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src, 0, src, 32, src0, src1); - DUP2_ARG3(__lasx_xvshuf_b, src0, src0, shuffler, src1, src1, shuffler, src0, - src1); - src0 = __lasx_xvpermi_q(src0, src0, 0x01); - src1 = __lasx_xvpermi_q(src1, src1, 0x01); - __lasx_xvst(src1, dst, 0); - __lasx_xvst(src0, dst, 32); - dst += 64; - src -= 64; - } +void MirrorRow_LASX(const uint8_t* src_y, uint8_t* dst_y, int width) { + static const uint8_t kRev[16] = {15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0}; + src_y += width - 32; + asm volatile( + "vld $vr1, %[rev], 0\n\t" + "xvpermi.q $xr1, $xr1, 0x00\n\t" + "1:\n\t" + "vld $vr0, %[src], 16\n\t" + "vld $vr31, %[src], 0\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvshuf.b $xr0, $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [rev] "r"(kRev) + : "memory", "$xr0", "$xr1", "$xr31"); } void MirrorUVRow_LASX(const uint8_t* src_uv, uint8_t* dst_uv, int width) { - int x; - int len = width / 16; - __m256i src, dst; - __m256i shuffler = {0x0004000500060007, 0x0000000100020003, - 0x0004000500060007, 0x0000000100020003}; - - src_uv += (width - 16) << 1; - for (x = 0; x < len; x++) { - src = __lasx_xvld(src_uv, 0); - dst = __lasx_xvshuf_h(shuffler, src, src); - dst = __lasx_xvpermi_q(dst, dst, 0x01); - __lasx_xvst(dst, dst_uv, 0); - src_uv -= 32; - dst_uv += 32; - } + static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1}; + src_uv += width * 2 - 32; + asm volatile( + "vld $vr1, %[rev], 0\n\t" + "xvpermi.q $xr1, $xr1, 0x00\n\t" + "1:\n\t" + "vld $vr0, %[src], 16\n\t" + "vld $vr31, %[src], 0\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvshuf.b $xr0, $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [cnt] "+r"(width) + : [rev] "r"(kRevUV) + : "memory", "$xr0", "$xr1", "$xr31"); } -void ARGBMirrorRow_LASX(const uint8_t* src, uint8_t* dst, int width) { - int x; - int len = width / 16; - __m256i src0, src1; - __m256i dst0, dst1; - __m256i shuffler = {0x0B0A09080F0E0D0C, 0x0302010007060504, - 0x0B0A09080F0E0D0C, 0x0302010007060504}; - src += (width * 4) - 64; - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src, 0, src, 32, src0, src1); - DUP2_ARG3(__lasx_xvshuf_b, src0, src0, shuffler, src1, src1, shuffler, src0, - src1); - dst1 = __lasx_xvpermi_q(src0, src0, 0x01); - dst0 = __lasx_xvpermi_q(src1, src1, 0x01); - __lasx_xvst(dst0, dst, 0); - __lasx_xvst(dst1, dst, 32); - dst += 64; - src -= 64; - } +void ARGBMirrorRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + src_argb += width * 4 - 32; + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 16\n\t" + "vld $vr31, %[src], 0\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvshuf4i.w $xr0, $xr0, 0x1b\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr31"); } -void CopyRow_LASX(const uint8_t* src, uint8_t* dst, int width) { - int x; - int len = width / 64; - __m256i src0, src1; +void MirrorSplitUVRow_LASX(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1}; + src_uv += width * 2 - 64; + asm volatile( + "vld $vr4, %[rev], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "1:\n\t" + "vld $vr0, %[src], 48\n\t" + "vld $vr31, %[src], 16\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 32\n\t" + "vld $vr31, %[src], 0\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvshuf.b $xr0, $xr0, $xr0, $xr4\n\t" + "xvshuf.b $xr1, $xr1, $xr1, $xr4\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvst $xr2, %[du], 0\n\t" + "xvst $xr3, %[dv], 0\n\t" + "addi.d %[src], %[src], -64\n\t" + "addi.d %[du], %[du], 32\n\t" + "addi.d %[dv], %[dv], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : + [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width) + : [rev] "r"(kRevUV) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); +} - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src, 0, src, 32, src0, src1); - __lasx_xvst(src0, dst, 0); - __lasx_xvst(src1, dst, 32); - src += 64; - dst += 64; - } +void SplitUVRow_LASX(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvst $xr2, %[du], 0\n\t" + "xvst $xr3, %[dv], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[du], %[du], 32\n\t" + "addi.d %[dv], %[dv], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : + [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); +} + +void MergeUVRow_LASX(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[su], 0\n\t" + "xvld $xr1, %[sv], 0\n\t" + "xvilvl.b $xr2, $xr1, $xr0\n\t" + "xvilvh.b $xr3, $xr1, $xr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr2, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr3, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr3, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[su], %[su], 32\n\t" + "addi.d %[sv], %[sv], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : + [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); +} + +void ARGBExtractAlphaRow_LASX(const uint8_t* src_argb, + uint8_t* dst_a, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickod.b $xr0, $xr1, $xr0\n\t" + "xvpickod.b $xr0, $xr0, $xr0\n\t" + "xvpermi.d $xr31, $xr0, 0xd8\n\t" + "vst $vr31, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_a), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr31"); } void I422ToYUY2Row_LASX(const uint8_t* src_y, @@ -308,29 +238,32 @@ void I422ToYUY2Row_LASX(const uint8_t* src_y, const uint8_t* src_v, uint8_t* dst_yuy2, int width) { - int x; - int len = width / 32; - __m256i src_u0, src_v0, src_y0, vec_uv0; - __m256i vec_yuy2_0, vec_yuy2_1; - __m256i dst_yuy2_0, dst_yuy2_1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_u, 0, src_v, 0, src_u0, src_v0); - src_y0 = __lasx_xvld(src_y, 0); - src_u0 = __lasx_xvpermi_d(src_u0, 0xD8); - src_v0 = __lasx_xvpermi_d(src_v0, 0xD8); - vec_uv0 = __lasx_xvilvl_b(src_v0, src_u0); - vec_yuy2_0 = __lasx_xvilvl_b(vec_uv0, src_y0); - vec_yuy2_1 = __lasx_xvilvh_b(vec_uv0, src_y0); - dst_yuy2_0 = __lasx_xvpermi_q(vec_yuy2_1, vec_yuy2_0, 0x20); - dst_yuy2_1 = __lasx_xvpermi_q(vec_yuy2_1, vec_yuy2_0, 0x31); - __lasx_xvst(dst_yuy2_0, dst_yuy2, 0); - __lasx_xvst(dst_yuy2_1, dst_yuy2, 32); - src_u += 16; - src_v += 16; - src_y += 32; - dst_yuy2 += 64; - } + asm volatile( + "1:\n\t" + "xvld $xr0, %[sy], 0\n\t" + "vld $vr1, %[su], 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "vld $vr2, %[sv], 0\n\t" + "xvpermi.d $xr2, $xr2, 0xd8\n\t" + "xvilvl.b $xr3, $xr2, $xr1\n\t" + "xvilvl.b $xr4, $xr3, $xr0\n\t" + "xvilvh.b $xr5, $xr3, $xr0\n\t" + "vst $vr4, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr4, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr5, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr5, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 32\n\t" + "addi.d %[su], %[su], 16\n\t" + "addi.d %[sv], %[sv], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_yuy2), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31"); } void I422ToUYVYRow_LASX(const uint8_t* src_y, @@ -338,29 +271,214 @@ void I422ToUYVYRow_LASX(const uint8_t* src_y, const uint8_t* src_v, uint8_t* dst_uyvy, int width) { - int x; - int len = width / 32; - __m256i src_u0, src_v0, src_y0, vec_uv0; - __m256i vec_uyvy0, vec_uyvy1; - __m256i dst_uyvy0, dst_uyvy1; + asm volatile( + "1:\n\t" + "xvld $xr0, %[sy], 0\n\t" + "vld $vr1, %[su], 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "vld $vr2, %[sv], 0\n\t" + "xvpermi.d $xr2, $xr2, 0xd8\n\t" + "xvilvl.b $xr3, $xr2, $xr1\n\t" + "xvilvl.b $xr4, $xr0, $xr3\n\t" + "xvilvh.b $xr5, $xr0, $xr3\n\t" + "vst $vr4, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr4, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr5, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr5, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 32\n\t" + "addi.d %[su], %[su], 16\n\t" + "addi.d %[sv], %[sv], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_uyvy), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31"); +} - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_u, 0, src_v, 0, src_u0, src_v0); - src_y0 = __lasx_xvld(src_y, 0); - src_u0 = __lasx_xvpermi_d(src_u0, 0xD8); - src_v0 = __lasx_xvpermi_d(src_v0, 0xD8); - vec_uv0 = __lasx_xvilvl_b(src_v0, src_u0); - vec_uyvy0 = __lasx_xvilvl_b(src_y0, vec_uv0); - vec_uyvy1 = __lasx_xvilvh_b(src_y0, vec_uv0); - dst_uyvy0 = __lasx_xvpermi_q(vec_uyvy1, vec_uyvy0, 0x20); - dst_uyvy1 = __lasx_xvpermi_q(vec_uyvy1, vec_uyvy0, 0x31); - __lasx_xvst(dst_uyvy0, dst_uyvy, 0); - __lasx_xvst(dst_uyvy1, dst_uyvy, 32); - src_u += 16; - src_v += 16; - src_y += 32; - dst_uyvy += 64; - } +void YUY2ToYRow_LASX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr31"); +} + +void UYVYToYRow_LASX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickod.b $xr2, $xr1, $xr0\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr31"); +} + +void YUY2ToUV422Row_LASX(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickod.b $xr2, $xr1, $xr0\n\t" + "xvpickev.b $xr3, $xr2, $xr2\n\t" + "xvpickod.b $xr4, $xr2, $xr2\n\t" + "xvpermi.d $xr31, $xr3, 0xd8\n\t" + "vst $vr31, %[du], 0\n\t" + "xvpermi.d $xr31, $xr4, 0xd8\n\t" + "vst $vr31, %[dv], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); +} + +void UYVYToUV422Row_LASX(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickev.b $xr3, $xr2, $xr2\n\t" + "xvpickod.b $xr4, $xr2, $xr2\n\t" + "xvpermi.d $xr31, $xr3, 0xd8\n\t" + "vst $vr31, %[du], 0\n\t" + "xvpermi.d $xr31, $xr4, 0xd8\n\t" + "vst $vr31, %[dv], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); +} + +void YUY2ToUVRow_LASX(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_yuy2 = src_yuy2 + src_stride_yuy2; + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr31, %[s0], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[s0], 16\n\t" + "vld $vr31, %[s0], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "vld $vr2, %[s1], 0\n\t" + "vld $vr31, %[s1], 32\n\t" + "xvpermi.q $xr2, $xr31, 0x02\n\t" + "vld $vr3, %[s1], 16\n\t" + "vld $vr31, %[s1], 48\n\t" + "xvpermi.q $xr3, $xr31, 0x02\n\t" + "xvpickod.b $xr0, $xr1, $xr0\n\t" + "xvpickod.b $xr2, $xr3, $xr2\n\t" + "xvavgr.bu $xr0, $xr0, $xr2\n\t" + "xvpickev.b $xr1, $xr0, $xr0\n\t" + "xvpickod.b $xr2, $xr0, $xr0\n\t" + "xvpermi.d $xr31, $xr1, 0xd8\n\t" + "vst $vr31, %[du], 0\n\t" + "xvpermi.d $xr31, $xr2, 0xd8\n\t" + "vst $vr31, %[dv], 0\n\t" + "addi.d %[s0], %[s0], 64\n\t" + "addi.d %[s1], %[s1], 64\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_yuy2), [s1] "+r"(next_yuy2), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); +} + +void UYVYToUVRow_LASX(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_uyvy = src_uyvy + src_stride_uyvy; + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr31, %[s0], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[s0], 16\n\t" + "vld $vr31, %[s0], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "vld $vr2, %[s1], 0\n\t" + "vld $vr31, %[s1], 32\n\t" + "xvpermi.q $xr2, $xr31, 0x02\n\t" + "vld $vr3, %[s1], 16\n\t" + "vld $vr31, %[s1], 48\n\t" + "xvpermi.q $xr3, $xr31, 0x02\n\t" + "xvpickev.b $xr0, $xr1, $xr0\n\t" + "xvpickev.b $xr2, $xr3, $xr2\n\t" + "xvavgr.bu $xr0, $xr0, $xr2\n\t" + "xvpickev.b $xr1, $xr0, $xr0\n\t" + "xvpickod.b $xr2, $xr0, $xr0\n\t" + "xvpermi.d $xr31, $xr1, 0xd8\n\t" + "vst $vr31, %[du], 0\n\t" + "xvpermi.d $xr31, $xr2, 0xd8\n\t" + "vst $vr31, %[dv], 0\n\t" + "addi.d %[s0], %[s0], 64\n\t" + "addi.d %[s1], %[s1], 64\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_uyvy), [s1] "+r"(next_uyvy), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); } void I422ToARGBRow_LASX(const uint8_t* src_y, @@ -369,58 +487,124 @@ void I422ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 32; - __m256i vec_yb, vec_yg, vec_ub, vec_ug, vec_vr, vec_vg; - __m256i vec_ubvr, vec_ugvg; - __m256i alpha = __lasx_xvldi(0xFF); - __m256i const_0x80 = __lasx_xvldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_ubvr = __lasx_xvilvl_h(vec_ub, vec_vr); - vec_ugvg = __lasx_xvilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m256i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - STOREARGB_D(alpha, alpha, r_l, r_h, g_l, g_h, b_l, b_h, dst_argb); - src_y += 32; - src_u += 16; - src_v += 16; - } + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvldrepl.d $xr10, %[su], 0\n\t" + "xvshuf4i.w $xr31, $xr10, 0x01\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "xvldrepl.d $xr11, %[sv], 0\n\t" + "xvshuf4i.w $xr31, $xr11, 0x01\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvilvl.b $xr10, $xr10, $xr10\n\t" + "xvilvl.b $xr11, $xr11, $xr11\n\t" + "xvilvl.b $xr1, $xr11, $xr10\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr3, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr4\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22", + "$xr23", "$xr24", "$xr31"); } void I422ToRGBARow_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, - uint8_t* dst_rgba, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 32; - __m256i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_ubvr, vec_ugvg; - __m256i alpha = __lasx_xvldi(0xFF); - __m256i const_0x80 = __lasx_xvldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_ubvr = __lasx_xvilvl_h(vec_ub, vec_vr); - vec_ugvg = __lasx_xvilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m256i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - STOREARGB_D(r_l, r_h, g_l, g_h, b_l, b_h, alpha, alpha, dst_rgba); - src_y += 32; - src_u += 16; - src_v += 16; - } + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvldrepl.d $xr10, %[su], 0\n\t" + "xvshuf4i.w $xr31, $xr10, 0x01\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "xvldrepl.d $xr11, %[sv], 0\n\t" + "xvshuf4i.w $xr31, $xr11, 0x01\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvilvl.b $xr10, $xr10, $xr10\n\t" + "xvilvl.b $xr11, $xr11, $xr11\n\t" + "xvilvl.b $xr1, $xr11, $xr10\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr2, $xr5\n\t" + "xvilvl.b $xr7, $xr4, $xr3\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22", + "$xr23", "$xr24", "$xr31"); } void I422AlphaToARGBRow_LASX(const uint8_t* src_y, @@ -430,41 +614,63 @@ void I422AlphaToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 32; - int res = width & 31; - __m256i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_ubvr, vec_ugvg; - __m256i zero = __lasx_xvldi(0); - __m256i const_0x80 = __lasx_xvldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_ubvr = __lasx_xvilvl_h(vec_ub, vec_vr); - vec_ugvg = __lasx_xvilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m256i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h, a_l, a_h; - - y = __lasx_xvld(src_a, 0); - a_l = __lasx_xvilvl_b(zero, y); - a_h = __lasx_xvilvh_b(zero, y); - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - STOREARGB_D(a_l, a_h, r_l, r_h, g_l, g_h, b_l, b_h, dst_argb); - src_y += 32; - src_u += 16; - src_v += 16; - src_a += 32; - } - if (res) { - __m256i y, uv, r, g, b, a; - a = __lasx_xvld(src_a, 0); - a = __lasx_vext2xv_hu_bu(a); - READYUV422(src_y, src_u, src_v, y, uv); - YUVTORGB(y, uv, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b, g, r); - STOREARGB(a, r, g, b, dst_argb); - } + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvldrepl.d $xr10, %[su], 0\n\t" + "xvshuf4i.w $xr31, $xr10, 0x01\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "xvldrepl.d $xr11, %[sv], 0\n\t" + "xvshuf4i.w $xr31, $xr11, 0x01\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvilvl.b $xr10, $xr10, $xr10\n\t" + "xvilvl.b $xr11, $xr11, $xr11\n\t" + "xvilvl.b $xr1, $xr11, $xr10\n\t" + "vld $vr5, %[sa], 0\n\t" + "xvpermi.d $xr5, $xr5, 0xd8\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr3, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr4\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[sa], %[sa], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width), [sa] "+r"(src_a) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22", + "$xr23", "$xr24", "$xr31"); } void I422ToRGB24Row_LASX(const uint8_t* src_y, @@ -472,1371 +678,282 @@ void I422ToRGB24Row_LASX(const uint8_t* src_y, const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, - int32_t width) { - int x; - int len = width / 32; - __m256i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_ubvr, vec_ugvg; - __m256i const_0x80 = __lasx_xvldi(0x80); - __m256i shuffler0 = {0x0504120302100100, 0x0A18090816070614, - 0x0504120302100100, 0x0A18090816070614}; - __m256i shuffler1 = {0x1E0F0E1C0D0C1A0B, 0x1E0F0E1C0D0C1A0B, - 0x1E0F0E1C0D0C1A0B, 0x1E0F0E1C0D0C1A0B}; - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_ubvr = __lasx_xvilvl_h(vec_ub, vec_vr); - vec_ugvg = __lasx_xvilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m256i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - __m256i temp0, temp1, temp2, temp3; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - temp0 = __lasx_xvpackev_b(g_l, b_l); - temp1 = __lasx_xvpackev_b(g_h, b_h); - DUP4_ARG3(__lasx_xvshuf_b, r_l, temp0, shuffler1, r_h, temp1, shuffler1, - r_l, temp0, shuffler0, r_h, temp1, shuffler0, temp2, temp3, temp0, - temp1); - - b_l = __lasx_xvilvl_d(temp1, temp2); - b_h = __lasx_xvilvh_d(temp3, temp1); - temp1 = __lasx_xvpermi_q(b_l, temp0, 0x20); - temp2 = __lasx_xvpermi_q(temp0, b_h, 0x30); - temp3 = __lasx_xvpermi_q(b_h, b_l, 0x31); - __lasx_xvst(temp1, dst_argb, 0); - __lasx_xvst(temp2, dst_argb, 32); - __lasx_xvst(temp3, dst_argb, 64); - dst_argb += 96; - src_y += 32; - src_u += 16; - src_v += 16; - } + int width) { + static const uint8_t kShuf0[16] = {0, 1, 2, 4, 5, 6, 8, 9, + 10, 12, 13, 14, 16, 17, 18, 20}; + static const uint8_t kShuf1[16] = {21, 22, 24, 25, 26, 28, 29, 30, + 0, 0, 0, 0, 0, 0, 0, 0}; + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "vld $vr12, %[s0], 0\n\t" + "xvpermi.q $xr12, $xr12, 0x00\n\t" + "vld $vr13, %[s1], 0\n\t" + "xvpermi.q $xr13, $xr13, 0x00\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvldrepl.d $xr10, %[su], 0\n\t" + "xvshuf4i.w $xr31, $xr10, 0x01\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "xvldrepl.d $xr11, %[sv], 0\n\t" + "xvshuf4i.w $xr31, $xr11, 0x01\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvilvl.b $xr10, $xr10, $xr10\n\t" + "xvilvl.b $xr11, $xr11, $xr11\n\t" + "xvilvl.b $xr1, $xr11, $xr10\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr3, $xr2\n\t" + "xvilvl.b $xr7, $xr4, $xr4\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "xvshuf.b $xr10, $xr9, $xr8, $xr12\n\t" + "xvshuf.b $xr11, $xr9, $xr8, $xr13\n\t" + "vst $vr10, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr10, 0x01\n\t" + "vst $vr31, %[dst], 24\n\t" + "vstelm.d $vr11, %[dst], 16, 0\n\t" + "xvpermi.q $xr31, $xr11, 0x01\n\t" + "vstelm.d $vr31, %[dst], 40, 0\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 48\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants), [s0] "r"(kShuf0), [s1] "r"(kShuf1) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr7", + "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr20", "$xr21", + "$xr22", "$xr23", "$xr24", "$xr31"); } -// TODO(fbarchard): Consider AND instead of shift to isolate 5 upper bits of R. void I422ToRGB565Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, - uint8_t* dst_rgb565, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 32; - __m256i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_ubvr, vec_ugvg; - __m256i const_0x80 = __lasx_xvldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_ubvr = __lasx_xvilvl_h(vec_ub, vec_vr); - vec_ugvg = __lasx_xvilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m256i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - __m256i dst_l, dst_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - b_l = __lasx_xvsrli_h(b_l, 3); - b_h = __lasx_xvsrli_h(b_h, 3); - g_l = __lasx_xvsrli_h(g_l, 2); - g_h = __lasx_xvsrli_h(g_h, 2); - r_l = __lasx_xvsrli_h(r_l, 3); - r_h = __lasx_xvsrli_h(r_h, 3); - r_l = __lasx_xvslli_h(r_l, 11); - r_h = __lasx_xvslli_h(r_h, 11); - g_l = __lasx_xvslli_h(g_l, 5); - g_h = __lasx_xvslli_h(g_h, 5); - r_l = __lasx_xvor_v(r_l, g_l); - r_l = __lasx_xvor_v(r_l, b_l); - r_h = __lasx_xvor_v(r_h, g_h); - r_h = __lasx_xvor_v(r_h, b_h); - dst_l = __lasx_xvpermi_q(r_h, r_l, 0x20); - dst_h = __lasx_xvpermi_q(r_h, r_l, 0x31); - __lasx_xvst(dst_l, dst_rgb565, 0); - __lasx_xvst(dst_h, dst_rgb565, 32); - dst_rgb565 += 64; - src_y += 32; - src_u += 16; - src_v += 16; - } + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr14, 31\n\t" + "xvsrli.h $xr14, $xr14, 8\n\t" + "xvldi $xr15, 63\n\t" + "xvsrli.h $xr15, $xr15, 8\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvldrepl.d $xr10, %[su], 0\n\t" + "xvshuf4i.w $xr31, $xr10, 0x01\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "xvldrepl.d $xr11, %[sv], 0\n\t" + "xvshuf4i.w $xr31, $xr11, 0x01\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvilvl.b $xr10, $xr10, $xr10\n\t" + "xvilvl.b $xr11, $xr11, $xr11\n\t" + "xvilvl.b $xr1, $xr11, $xr10\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvsrli.b $xr2, $xr2, 3\n\t" + "xvsrli.b $xr3, $xr3, 2\n\t" + "xvsrli.b $xr4, $xr4, 3\n\t" + "xvilvl.b $xr2, $xr2, $xr2\n\t" + "xvilvl.b $xr3, $xr3, $xr3\n\t" + "xvilvl.b $xr4, $xr4, $xr4\n\t" + "xvand.v $xr2, $xr2, $xr14\n\t" + "xvand.v $xr3, $xr3, $xr15\n\t" + "xvand.v $xr4, $xr4, $xr14\n\t" + "xvslli.h $xr3, $xr3, 5\n\t" + "xvslli.h $xr4, $xr4, 11\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvor.v $xr2, $xr2, $xr4\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11", + "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } -// TODO(fbarchard): Consider AND instead of shift to isolate 4 upper bits of G. void I422ToARGB4444Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, - uint8_t* dst_argb4444, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 32; - __m256i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_ubvr, vec_ugvg; - __m256i const_0x80 = __lasx_xvldi(0x80); - __m256i alpha = (__m256i)v4u64{0xF000F000F000F000, 0xF000F000F000F000, - 0xF000F000F000F000, 0xF000F000F000F000}; - __m256i mask = {0x00F000F000F000F0, 0x00F000F000F000F0, 0x00F000F000F000F0, - 0x00F000F000F000F0}; - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_ubvr = __lasx_xvilvl_h(vec_ub, vec_vr); - vec_ugvg = __lasx_xvilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m256i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - __m256i dst_l, dst_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - b_l = __lasx_xvsrli_h(b_l, 4); - b_h = __lasx_xvsrli_h(b_h, 4); - r_l = __lasx_xvsrli_h(r_l, 4); - r_h = __lasx_xvsrli_h(r_h, 4); - g_l = __lasx_xvand_v(g_l, mask); - g_h = __lasx_xvand_v(g_h, mask); - r_l = __lasx_xvslli_h(r_l, 8); - r_h = __lasx_xvslli_h(r_h, 8); - r_l = __lasx_xvor_v(r_l, alpha); - r_h = __lasx_xvor_v(r_h, alpha); - r_l = __lasx_xvor_v(r_l, g_l); - r_h = __lasx_xvor_v(r_h, g_h); - r_l = __lasx_xvor_v(r_l, b_l); - r_h = __lasx_xvor_v(r_h, b_h); - dst_l = __lasx_xvpermi_q(r_h, r_l, 0x20); - dst_h = __lasx_xvpermi_q(r_h, r_l, 0x31); - __lasx_xvst(dst_l, dst_argb4444, 0); - __lasx_xvst(dst_h, dst_argb4444, 32); - dst_argb4444 += 64; - src_y += 32; - src_u += 16; - src_v += 16; - } + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr14, 15\n\t" + "xvsrli.h $xr14, $xr14, 8\n\t" + "xvldi $xr15, 240\n\t" + "xvsrli.h $xr15, $xr15, 8\n\t" + "xvslli.h $xr15, $xr15, 8\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvldrepl.d $xr10, %[su], 0\n\t" + "xvshuf4i.w $xr31, $xr10, 0x01\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "xvldrepl.d $xr11, %[sv], 0\n\t" + "xvshuf4i.w $xr31, $xr11, 0x01\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvilvl.b $xr10, $xr10, $xr10\n\t" + "xvilvl.b $xr11, $xr11, $xr11\n\t" + "xvilvl.b $xr1, $xr11, $xr10\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvsrli.b $xr2, $xr2, 4\n\t" + "xvsrli.b $xr3, $xr3, 4\n\t" + "xvsrli.b $xr4, $xr4, 4\n\t" + "xvilvl.b $xr2, $xr2, $xr2\n\t" + "xvilvl.b $xr3, $xr3, $xr3\n\t" + "xvilvl.b $xr4, $xr4, $xr4\n\t" + "xvand.v $xr2, $xr2, $xr14\n\t" + "xvand.v $xr3, $xr3, $xr14\n\t" + "xvand.v $xr4, $xr4, $xr14\n\t" + "xvslli.h $xr3, $xr3, 4\n\t" + "xvslli.h $xr4, $xr4, 8\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvor.v $xr2, $xr2, $xr4\n\t" + "xvor.v $xr2, $xr2, $xr15\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11", + "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } void I422ToARGB1555Row_LASX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, - uint8_t* dst_argb1555, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 32; - __m256i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_ubvr, vec_ugvg; - __m256i const_0x80 = __lasx_xvldi(0x80); - __m256i alpha = (__m256i)v4u64{0x8000800080008000, 0x8000800080008000, - 0x8000800080008000, 0x8000800080008000}; - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_ubvr = __lasx_xvilvl_h(vec_ub, vec_vr); - vec_ugvg = __lasx_xvilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m256i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - __m256i dst_l, dst_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - b_l = __lasx_xvsrli_h(b_l, 3); - b_h = __lasx_xvsrli_h(b_h, 3); - g_l = __lasx_xvsrli_h(g_l, 3); - g_h = __lasx_xvsrli_h(g_h, 3); - g_l = __lasx_xvslli_h(g_l, 5); - g_h = __lasx_xvslli_h(g_h, 5); - r_l = __lasx_xvsrli_h(r_l, 3); - r_h = __lasx_xvsrli_h(r_h, 3); - r_l = __lasx_xvslli_h(r_l, 10); - r_h = __lasx_xvslli_h(r_h, 10); - r_l = __lasx_xvor_v(r_l, alpha); - r_h = __lasx_xvor_v(r_h, alpha); - r_l = __lasx_xvor_v(r_l, g_l); - r_h = __lasx_xvor_v(r_h, g_h); - r_l = __lasx_xvor_v(r_l, b_l); - r_h = __lasx_xvor_v(r_h, b_h); - dst_l = __lasx_xvpermi_q(r_h, r_l, 0x20); - dst_h = __lasx_xvpermi_q(r_h, r_l, 0x31); - __lasx_xvst(dst_l, dst_argb1555, 0); - __lasx_xvst(dst_h, dst_argb1555, 32); - dst_argb1555 += 64; - src_y += 32; - src_u += 16; - src_v += 16; - } -} - -void YUY2ToYRow_LASX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { - int x; - int len = width / 32; - __m256i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_yuy2, 0, src_yuy2, 32, src0, src1); - dst0 = __lasx_xvpickev_b(src1, src0); - dst0 = __lasx_xvpermi_d(dst0, 0xD8); - __lasx_xvst(dst0, dst_y, 0); - src_yuy2 += 64; - dst_y += 32; - } -} - -void YUY2ToUVRow_LASX(const uint8_t* src_yuy2, - int src_stride_yuy2, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - const uint8_t* src_yuy2_next = src_yuy2 + src_stride_yuy2; - int x; - int len = width / 32; - __m256i src0, src1, src2, src3; - __m256i tmp0, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lasx_xvld, src_yuy2, 0, src_yuy2, 32, src_yuy2_next, 0, - src_yuy2_next, 32, src0, src1, src2, src3); - src0 = __lasx_xvpickod_b(src1, src0); - src1 = __lasx_xvpickod_b(src3, src2); - tmp0 = __lasx_xvavgr_bu(src1, src0); - tmp0 = __lasx_xvpermi_d(tmp0, 0xD8); - dst0 = __lasx_xvpickev_b(tmp0, tmp0); - dst1 = __lasx_xvpickod_b(tmp0, tmp0); - __lasx_xvstelm_d(dst0, dst_u, 0, 0); - __lasx_xvstelm_d(dst0, dst_u, 8, 2); - __lasx_xvstelm_d(dst1, dst_v, 0, 0); - __lasx_xvstelm_d(dst1, dst_v, 8, 2); - src_yuy2 += 64; - src_yuy2_next += 64; - dst_u += 16; - dst_v += 16; - } -} - -void YUY2ToUV422Row_LASX(const uint8_t* src_yuy2, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 32; - __m256i src0, src1, tmp0, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_yuy2, 0, src_yuy2, 32, src0, src1); - tmp0 = __lasx_xvpickod_b(src1, src0); - tmp0 = __lasx_xvpermi_d(tmp0, 0xD8); - dst0 = __lasx_xvpickev_b(tmp0, tmp0); - dst1 = __lasx_xvpickod_b(tmp0, tmp0); - __lasx_xvstelm_d(dst0, dst_u, 0, 0); - __lasx_xvstelm_d(dst0, dst_u, 8, 2); - __lasx_xvstelm_d(dst1, dst_v, 0, 0); - __lasx_xvstelm_d(dst1, dst_v, 8, 2); - src_yuy2 += 64; - dst_u += 16; - dst_v += 16; - } -} - -void UYVYToYRow_LASX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { - int x; - int len = width / 32; - __m256i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_uyvy, 0, src_uyvy, 32, src0, src1); - dst0 = __lasx_xvpickod_b(src1, src0); - dst0 = __lasx_xvpermi_d(dst0, 0xD8); - __lasx_xvst(dst0, dst_y, 0); - src_uyvy += 64; - dst_y += 32; - } -} - -void UYVYToUVRow_LASX(const uint8_t* src_uyvy, - int src_stride_uyvy, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - const uint8_t* src_uyvy_next = src_uyvy + src_stride_uyvy; - int x; - int len = width / 32; - __m256i src0, src1, src2, src3, tmp0, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lasx_xvld, src_uyvy, 0, src_uyvy, 32, src_uyvy_next, 0, - src_uyvy_next, 32, src0, src1, src2, src3); - src0 = __lasx_xvpickev_b(src1, src0); - src1 = __lasx_xvpickev_b(src3, src2); - tmp0 = __lasx_xvavgr_bu(src1, src0); - tmp0 = __lasx_xvpermi_d(tmp0, 0xD8); - dst0 = __lasx_xvpickev_b(tmp0, tmp0); - dst1 = __lasx_xvpickod_b(tmp0, tmp0); - __lasx_xvstelm_d(dst0, dst_u, 0, 0); - __lasx_xvstelm_d(dst0, dst_u, 8, 2); - __lasx_xvstelm_d(dst1, dst_v, 0, 0); - __lasx_xvstelm_d(dst1, dst_v, 8, 2); - src_uyvy += 64; - src_uyvy_next += 64; - dst_u += 16; - dst_v += 16; - } -} - -void UYVYToUV422Row_LASX(const uint8_t* src_uyvy, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 32; - __m256i src0, src1, tmp0, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_uyvy, 0, src_uyvy, 32, src0, src1); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp0 = __lasx_xvpermi_d(tmp0, 0xD8); - dst0 = __lasx_xvpickev_b(tmp0, tmp0); - dst1 = __lasx_xvpickod_b(tmp0, tmp0); - __lasx_xvstelm_d(dst0, dst_u, 0, 0); - __lasx_xvstelm_d(dst0, dst_u, 8, 2); - __lasx_xvstelm_d(dst1, dst_v, 0, 0); - __lasx_xvstelm_d(dst1, dst_v, 8, 2); - src_uyvy += 64; - dst_u += 16; - dst_v += 16; - } -} -void ARGBToRGB24Row_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - int x; - int len = (width / 32) - 1; - __m256i src0, src1, src2, src3; - __m256i tmp0, tmp1, tmp2, tmp3; - __m256i shuf = {0x0908060504020100, 0x000000000E0D0C0A, 0x0908060504020100, - 0x000000000E0D0C0A}; - __m256i control = {0x0000000100000000, 0x0000000400000002, 0x0000000600000005, - 0x0000000700000003}; - for (x = 0; x < len; x++) { - DUP4_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src_argb, 64, src_argb, - 96, src0, src1, src2, src3); - tmp0 = __lasx_xvshuf_b(src0, src0, shuf); - tmp1 = __lasx_xvshuf_b(src1, src1, shuf); - tmp2 = __lasx_xvshuf_b(src2, src2, shuf); - tmp3 = __lasx_xvshuf_b(src3, src3, shuf); - tmp0 = __lasx_xvperm_w(tmp0, control); - tmp1 = __lasx_xvperm_w(tmp1, control); - tmp2 = __lasx_xvperm_w(tmp2, control); - tmp3 = __lasx_xvperm_w(tmp3, control); - __lasx_xvst(tmp0, dst_rgb, 0); - __lasx_xvst(tmp1, dst_rgb, 24); - __lasx_xvst(tmp2, dst_rgb, 48); - __lasx_xvst(tmp3, dst_rgb, 72); - dst_rgb += 96; - src_argb += 128; - } - DUP4_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src_argb, 64, src_argb, 96, - src0, src1, src2, src3); - tmp0 = __lasx_xvshuf_b(src0, src0, shuf); - tmp1 = __lasx_xvshuf_b(src1, src1, shuf); - tmp2 = __lasx_xvshuf_b(src2, src2, shuf); - tmp3 = __lasx_xvshuf_b(src3, src3, shuf); - tmp0 = __lasx_xvperm_w(tmp0, control); - tmp1 = __lasx_xvperm_w(tmp1, control); - tmp2 = __lasx_xvperm_w(tmp2, control); - tmp3 = __lasx_xvperm_w(tmp3, control); - __lasx_xvst(tmp0, dst_rgb, 0); - __lasx_xvst(tmp1, dst_rgb, 24); - __lasx_xvst(tmp2, dst_rgb, 48); - dst_rgb += 72; - __lasx_xvstelm_d(tmp3, dst_rgb, 0, 0); - __lasx_xvstelm_d(tmp3, dst_rgb, 8, 1); - __lasx_xvstelm_d(tmp3, dst_rgb, 16, 2); -} - -void ARGBToRAWRow_LASX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - int x; - int len = (width / 32) - 1; - __m256i src0, src1, src2, src3; - __m256i tmp0, tmp1, tmp2, tmp3; - __m256i shuf = {0x090A040506000102, 0x000000000C0D0E08, 0x090A040506000102, - 0x000000000C0D0E08}; - __m256i control = {0x0000000100000000, 0x0000000400000002, 0x0000000600000005, - 0x0000000700000003}; - for (x = 0; x < len; x++) { - DUP4_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src_argb, 64, src_argb, - 96, src0, src1, src2, src3); - tmp0 = __lasx_xvshuf_b(src0, src0, shuf); - tmp1 = __lasx_xvshuf_b(src1, src1, shuf); - tmp2 = __lasx_xvshuf_b(src2, src2, shuf); - tmp3 = __lasx_xvshuf_b(src3, src3, shuf); - tmp0 = __lasx_xvperm_w(tmp0, control); - tmp1 = __lasx_xvperm_w(tmp1, control); - tmp2 = __lasx_xvperm_w(tmp2, control); - tmp3 = __lasx_xvperm_w(tmp3, control); - __lasx_xvst(tmp0, dst_rgb, 0); - __lasx_xvst(tmp1, dst_rgb, 24); - __lasx_xvst(tmp2, dst_rgb, 48); - __lasx_xvst(tmp3, dst_rgb, 72); - dst_rgb += 96; - src_argb += 128; - } - DUP4_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src_argb, 64, src_argb, 96, - src0, src1, src2, src3); - tmp0 = __lasx_xvshuf_b(src0, src0, shuf); - tmp1 = __lasx_xvshuf_b(src1, src1, shuf); - tmp2 = __lasx_xvshuf_b(src2, src2, shuf); - tmp3 = __lasx_xvshuf_b(src3, src3, shuf); - tmp0 = __lasx_xvperm_w(tmp0, control); - tmp1 = __lasx_xvperm_w(tmp1, control); - tmp2 = __lasx_xvperm_w(tmp2, control); - tmp3 = __lasx_xvperm_w(tmp3, control); - __lasx_xvst(tmp0, dst_rgb, 0); - __lasx_xvst(tmp1, dst_rgb, 24); - __lasx_xvst(tmp2, dst_rgb, 48); - dst_rgb += 72; - __lasx_xvstelm_d(tmp3, dst_rgb, 0, 0); - __lasx_xvstelm_d(tmp3, dst_rgb, 8, 1); - __lasx_xvstelm_d(tmp3, dst_rgb, 16, 2); -} - -void ARGBToRGB565Row_LASX(const uint8_t* src_argb, - uint8_t* dst_rgb, - int width) { - int x; - int len = width / 16; - __m256i zero = __lasx_xvldi(0); - __m256i src0, src1, tmp0, tmp1, dst0; - __m256i shift = {0x0300030003000300, 0x0300030003000300, 0x0300030003000300, - 0x0300030003000300}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src0, src1); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - tmp0 = __lasx_xvsrli_b(tmp0, 3); - tmp1 = __lasx_xvpackev_b(zero, tmp1); - tmp1 = __lasx_xvsrli_h(tmp1, 2); - tmp0 = __lasx_xvsll_b(tmp0, shift); - tmp1 = __lasx_xvslli_h(tmp1, 5); - dst0 = __lasx_xvor_v(tmp0, tmp1); - dst0 = __lasx_xvpermi_d(dst0, 0xD8); - __lasx_xvst(dst0, dst_rgb, 0); - dst_rgb += 32; - src_argb += 64; - } -} - -void ARGBToARGB1555Row_LASX(const uint8_t* src_argb, - uint8_t* dst_rgb, - int width) { - int x; - int len = width / 16; - __m256i zero = __lasx_xvldi(0); - __m256i src0, src1, tmp0, tmp1, tmp2, tmp3, dst0; - __m256i shift1 = {0x0703070307030703, 0x0703070307030703, 0x0703070307030703, - 0x0703070307030703}; - __m256i shift2 = {0x0200020002000200, 0x0200020002000200, 0x0200020002000200, - 0x0200020002000200}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src0, src1); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - tmp0 = __lasx_xvsrli_b(tmp0, 3); - tmp1 = __lasx_xvsrl_b(tmp1, shift1); - tmp0 = __lasx_xvsll_b(tmp0, shift2); - tmp2 = __lasx_xvpackev_b(zero, tmp1); - tmp3 = __lasx_xvpackod_b(zero, tmp1); - tmp2 = __lasx_xvslli_h(tmp2, 5); - tmp3 = __lasx_xvslli_h(tmp3, 15); - dst0 = __lasx_xvor_v(tmp0, tmp2); - dst0 = __lasx_xvor_v(dst0, tmp3); - dst0 = __lasx_xvpermi_d(dst0, 0xD8); - __lasx_xvst(dst0, dst_rgb, 0); - dst_rgb += 32; - src_argb += 64; - } -} - -void ARGBToARGB4444Row_LASX(const uint8_t* src_argb, - uint8_t* dst_rgb, - int width) { - int x; - int len = width / 16; - __m256i src0, src1, tmp0, tmp1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src0, src1); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - tmp1 = __lasx_xvandi_b(tmp1, 0xF0); - tmp0 = __lasx_xvsrli_b(tmp0, 4); - dst0 = __lasx_xvor_v(tmp1, tmp0); - dst0 = __lasx_xvpermi_d(dst0, 0xD8); - __lasx_xvst(dst0, dst_rgb, 0); - dst_rgb += 32; - src_argb += 64; - } -} - -void ARGBMultiplyRow_LASX(const uint8_t* src_argb0, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 8; - __m256i half = __lasx_xvldi(0x480); - __m256i src0, src1, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb0, 0, src_argb1, 0, src0, src1); - dst0 = __lasx_xvmaddwev_h_bu(half, src0, src1); - dst1 = __lasx_xvmaddwod_h_bu(half, src0, src1); - dst0 = __lasx_xvpackod_b(dst1, dst0); - __lasx_xvst(dst0, dst_argb, 0); - src_argb0 += 32; - src_argb1 += 32; - dst_argb += 32; - } -} - -void ARGBAddRow_LASX(const uint8_t* src_argb0, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 8; - __m256i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb0, 0, src_argb1, 0, src0, src1); - dst0 = __lasx_xvsadd_bu(src0, src1); - __lasx_xvst(dst0, dst_argb, 0); - src_argb0 += 32; - src_argb1 += 32; - dst_argb += 32; - } -} - -void ARGBSubtractRow_LASX(const uint8_t* src_argb0, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 8; - __m256i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb0, 0, src_argb1, 0, src0, src1); - dst0 = __lasx_xvssub_bu(src0, src1); - __lasx_xvst(dst0, dst_argb, 0); - src_argb0 += 32; - src_argb1 += 32; - dst_argb += 32; - } -} - -void ARGBAttenuateRow_LASX(const uint8_t* src_argb, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 16; - __m256i src0, src1, tmp0, tmp1; - __m256i reg0, reg1, reg2, reg3, reg4, reg5; - __m256i b, g, r, a, dst0, dst1; - __m256i control = {0x0005000100040000, 0x0007000300060002, 0x0005000100040000, - 0x0007000300060002}; - __m256i zero = __lasx_xvldi(0); - __m256i const_add = __lasx_xvldi(0x8ff); - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src0, src1); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - b = __lasx_xvpackev_b(zero, tmp0); - r = __lasx_xvpackod_b(zero, tmp0); - g = __lasx_xvpackev_b(zero, tmp1); - a = __lasx_xvpackod_b(zero, tmp1); - reg0 = __lasx_xvmaddwev_w_hu(const_add, b, a); - reg1 = __lasx_xvmaddwod_w_hu(const_add, b, a); - reg2 = __lasx_xvmaddwev_w_hu(const_add, r, a); - reg3 = __lasx_xvmaddwod_w_hu(const_add, r, a); - reg4 = __lasx_xvmaddwev_w_hu(const_add, g, a); - reg5 = __lasx_xvmaddwod_w_hu(const_add, g, a); - reg0 = __lasx_xvssrani_h_w(reg1, reg0, 8); - reg2 = __lasx_xvssrani_h_w(reg3, reg2, 8); - reg4 = __lasx_xvssrani_h_w(reg5, reg4, 8); - reg0 = __lasx_xvshuf_h(control, reg0, reg0); - reg2 = __lasx_xvshuf_h(control, reg2, reg2); - reg4 = __lasx_xvshuf_h(control, reg4, reg4); - tmp0 = __lasx_xvpackev_b(reg4, reg0); - tmp1 = __lasx_xvpackev_b(a, reg2); - dst0 = __lasx_xvilvl_h(tmp1, tmp0); - dst1 = __lasx_xvilvh_h(tmp1, tmp0); - __lasx_xvst(dst0, dst_argb, 0); - __lasx_xvst(dst1, dst_argb, 32); - dst_argb += 64; - src_argb += 64; - } -} - -void ARGBToRGB565DitherRow_LASX(const uint8_t* src_argb, - uint8_t* dst_rgb, - uint32_t dither4, - int width) { - int x; - int len = width / 16; - __m256i src0, src1, tmp0, tmp1, dst0; - __m256i b, g, r; - __m256i zero = __lasx_xvldi(0); - __m256i vec_dither = __lasx_xvldrepl_w(&dither4, 0); - - vec_dither = __lasx_xvilvl_b(zero, vec_dither); - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src0, src1); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - b = __lasx_xvpackev_b(zero, tmp0); - r = __lasx_xvpackod_b(zero, tmp0); - g = __lasx_xvpackev_b(zero, tmp1); - b = __lasx_xvadd_h(b, vec_dither); - g = __lasx_xvadd_h(g, vec_dither); - r = __lasx_xvadd_h(r, vec_dither); - DUP2_ARG1(__lasx_xvclip255_h, b, g, b, g); - r = __lasx_xvclip255_h(r); - b = __lasx_xvsrai_h(b, 3); - g = __lasx_xvsrai_h(g, 2); - r = __lasx_xvsrai_h(r, 3); - g = __lasx_xvslli_h(g, 5); - r = __lasx_xvslli_h(r, 11); - dst0 = __lasx_xvor_v(b, g); - dst0 = __lasx_xvor_v(dst0, r); - dst0 = __lasx_xvpermi_d(dst0, 0xD8); - __lasx_xvst(dst0, dst_rgb, 0); - src_argb += 64; - dst_rgb += 32; - } -} - -void ARGBShuffleRow_LASX(const uint8_t* src_argb, - uint8_t* dst_argb, - const uint8_t* shuffler, - int width) { - int x; - int len = width / 16; - __m256i src0, src1, dst0, dst1; - __m256i shuf = {0x0404040400000000, 0x0C0C0C0C08080808, 0x0404040400000000, - 0x0C0C0C0C08080808}; - __m256i temp = __lasx_xvldrepl_w(shuffler, 0); - - shuf = __lasx_xvadd_b(shuf, temp); - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src0, src1); - dst0 = __lasx_xvshuf_b(src0, src0, shuf); - dst1 = __lasx_xvshuf_b(src1, src1, shuf); - __lasx_xvst(dst0, dst_argb, 0); - __lasx_xvst(dst1, dst_argb, 32); - src_argb += 64; - dst_argb += 64; - } -} - -void ARGBShadeRow_LASX(const uint8_t* src_argb, - uint8_t* dst_argb, - int width, - uint32_t value) { - int x; - int len = width / 8; - __m256i src0, dst0, tmp0, tmp1; - __m256i vec_value = __lasx_xvreplgr2vr_w(value); - - vec_value = __lasx_xvilvl_b(vec_value, vec_value); - for (x = 0; x < len; x++) { - src0 = __lasx_xvld(src_argb, 0); - tmp0 = __lasx_xvilvl_b(src0, src0); - tmp1 = __lasx_xvilvh_b(src0, src0); - tmp0 = __lasx_xvmuh_hu(tmp0, vec_value); - tmp1 = __lasx_xvmuh_hu(tmp1, vec_value); - dst0 = __lasx_xvpickod_b(tmp1, tmp0); - __lasx_xvst(dst0, dst_argb, 0); - src_argb += 32; - dst_argb += 32; - } -} - -void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { - int x; - int len = width / 16; - __m256i src0, src1, tmp0, tmp1; - __m256i reg0, reg1, reg2, dst0, dst1; - __m256i const_128 = __lasx_xvldi(0x480); - __m256i const_150 = __lasx_xvldi(0x96); - __m256i const_br = {0x4D1D4D1D4D1D4D1D, 0x4D1D4D1D4D1D4D1D, - 0x4D1D4D1D4D1D4D1D, 0x4D1D4D1D4D1D4D1D}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, src_argb, 0, src_argb, 32, src0, src1); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - reg0 = __lasx_xvdp2_h_bu(tmp0, const_br); - reg1 = __lasx_xvmaddwev_h_bu(const_128, tmp1, const_150); - reg2 = __lasx_xvadd_h(reg0, reg1); - tmp0 = __lasx_xvpackod_b(reg2, reg2); - tmp1 = __lasx_xvpackod_b(tmp1, reg2); - dst0 = __lasx_xvilvl_h(tmp1, tmp0); - dst1 = __lasx_xvilvh_h(tmp1, tmp0); - __lasx_xvst(dst0, dst_argb, 0); - __lasx_xvst(dst1, dst_argb, 32); - src_argb += 64; - dst_argb += 64; - } -} - -void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width) { - int x; - int len = width / 16; - __m256i src0, src1, tmp0, tmp1; - __m256i reg0, reg1, spb, spg, spr; - __m256i dst0, dst1; - __m256i spb_g = __lasx_xvldi(68); - __m256i spg_g = __lasx_xvldi(88); - __m256i spr_g = __lasx_xvldi(98); - __m256i spb_br = {0x2311231123112311, 0x2311231123112311, 0x2311231123112311, - 0x2311231123112311}; - __m256i spg_br = {0x2D162D162D162D16, 0x2D162D162D162D16, 0x2D162D162D162D16, - 0x2D162D162D162D16}; - __m256i spr_br = {0x3218321832183218, 0x3218321832183218, 0x3218321832183218, - 0x3218321832183218}; - __m256i shuff = {0x1706150413021100, 0x1F0E1D0C1B0A1908, 0x1706150413021100, - 0x1F0E1D0C1B0A1908}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lasx_xvld, dst_argb, 0, dst_argb, 32, src0, src1); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - DUP2_ARG2(__lasx_xvdp2_h_bu, tmp0, spb_br, tmp0, spg_br, spb, spg); - spr = __lasx_xvdp2_h_bu(tmp0, spr_br); - spb = __lasx_xvmaddwev_h_bu(spb, tmp1, spb_g); - spg = __lasx_xvmaddwev_h_bu(spg, tmp1, spg_g); - spr = __lasx_xvmaddwev_h_bu(spr, tmp1, spr_g); - spb = __lasx_xvsrli_h(spb, 7); - spg = __lasx_xvsrli_h(spg, 7); - spr = __lasx_xvsrli_h(spr, 7); - spg = __lasx_xvsat_hu(spg, 7); - spr = __lasx_xvsat_hu(spr, 7); - reg0 = __lasx_xvpackev_b(spg, spb); - reg1 = __lasx_xvshuf_b(tmp1, spr, shuff); - dst0 = __lasx_xvilvl_h(reg1, reg0); - dst1 = __lasx_xvilvh_h(reg1, reg0); - __lasx_xvst(dst0, dst_argb, 0); - __lasx_xvst(dst1, dst_argb, 32); - dst_argb += 64; - } -} - -void ARGB4444ToARGBRow_LASX(const uint8_t* src_argb4444, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 32; - __m256i src0, src1; - __m256i tmp0, tmp1, tmp2, tmp3; - __m256i reg0, reg1, reg2, reg3; - __m256i dst0, dst1, dst2, dst3; - - for (x = 0; x < len; x++) { - src0 = __lasx_xvld(src_argb4444, 0); - src1 = __lasx_xvld(src_argb4444, 32); - DUP4_ARG2(__lasx_xvandi_b, src0, 0x0F, src0, 0xF0, src1, 0x0F, src1, 0xF0, - tmp0, tmp1, tmp2, tmp3); - DUP2_ARG2(__lasx_xvslli_b, tmp0, 4, tmp2, 4, reg0, reg2); - DUP2_ARG2(__lasx_xvsrli_b, tmp1, 4, tmp3, 4, reg1, reg3); - DUP4_ARG2(__lasx_xvor_v, tmp0, reg0, tmp1, reg1, tmp2, reg2, tmp3, reg3, - tmp0, tmp1, tmp2, tmp3); - DUP2_ARG2(__lasx_xvilvl_b, tmp1, tmp0, tmp3, tmp2, reg0, reg2); - DUP2_ARG2(__lasx_xvilvh_b, tmp1, tmp0, tmp3, tmp2, reg1, reg3); - DUP4_ARG3(__lasx_xvpermi_q, reg1, reg0, 0x20, reg1, reg0, 0x31, reg3, reg2, - 0x20, reg3, reg2, 0x31, dst0, dst1, dst2, dst3); - __lasx_xvst(dst0, dst_argb, 0); - __lasx_xvst(dst1, dst_argb, 32); - __lasx_xvst(dst2, dst_argb, 64); - __lasx_xvst(dst3, dst_argb, 96); - src_argb4444 += 64; - dst_argb += 128; - } -} - -void ARGB1555ToARGBRow_LASX(const uint8_t* src_argb1555, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 32; - __m256i src0, src1; - __m256i tmp0, tmp1, tmpb, tmpg, tmpr, tmpa; - __m256i reg0, reg1, reg2, reg3; - __m256i dst0, dst1, dst2, dst3; - - for (x = 0; x < len; x++) { - src0 = __lasx_xvld(src_argb1555, 0); - src1 = __lasx_xvld(src_argb1555, 32); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - tmpb = __lasx_xvandi_b(tmp0, 0x1F); - tmpg = __lasx_xvsrli_b(tmp0, 5); - reg0 = __lasx_xvandi_b(tmp1, 0x03); - reg0 = __lasx_xvslli_b(reg0, 3); - tmpg = __lasx_xvor_v(tmpg, reg0); - reg1 = __lasx_xvandi_b(tmp1, 0x7C); - tmpr = __lasx_xvsrli_b(reg1, 2); - tmpa = __lasx_xvsrli_b(tmp1, 7); - tmpa = __lasx_xvneg_b(tmpa); - reg0 = __lasx_xvslli_b(tmpb, 3); - reg1 = __lasx_xvslli_b(tmpg, 3); - reg2 = __lasx_xvslli_b(tmpr, 3); - tmpb = __lasx_xvsrli_b(tmpb, 2); - tmpg = __lasx_xvsrli_b(tmpg, 2); - tmpr = __lasx_xvsrli_b(tmpr, 2); - tmpb = __lasx_xvor_v(reg0, tmpb); - tmpg = __lasx_xvor_v(reg1, tmpg); - tmpr = __lasx_xvor_v(reg2, tmpr); - DUP2_ARG2(__lasx_xvilvl_b, tmpg, tmpb, tmpa, tmpr, reg0, reg1); - DUP2_ARG2(__lasx_xvilvh_b, tmpg, tmpb, tmpa, tmpr, reg2, reg3); - dst0 = __lasx_xvilvl_h(reg1, reg0); - dst1 = __lasx_xvilvh_h(reg1, reg0); - dst2 = __lasx_xvilvl_h(reg3, reg2); - dst3 = __lasx_xvilvh_h(reg3, reg2); - DUP4_ARG3(__lasx_xvpermi_q, dst1, dst0, 0x20, dst1, dst0, 0x31, dst3, dst2, - 0x20, dst3, dst2, 0x31, reg0, reg1, reg2, reg3); - __lasx_xvst(reg0, dst_argb, 0); - __lasx_xvst(reg1, dst_argb, 32); - __lasx_xvst(reg2, dst_argb, 64); - __lasx_xvst(reg3, dst_argb, 96); - src_argb1555 += 64; - dst_argb += 128; - } -} - -void RGB565ToARGBRow_LASX(const uint8_t* src_rgb565, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 32; - __m256i src0, src1; - __m256i tmp0, tmp1, tmpb, tmpg, tmpr; - __m256i reg0, reg1, reg2, reg3, dst0, dst1, dst2, dst3; - __m256i alpha = __lasx_xvldi(0xFF); - - for (x = 0; x < len; x++) { - src0 = __lasx_xvld(src_rgb565, 0); - src1 = __lasx_xvld(src_rgb565, 32); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - tmpb = __lasx_xvandi_b(tmp0, 0x1F); - tmpr = __lasx_xvandi_b(tmp1, 0xF8); - reg1 = __lasx_xvandi_b(tmp1, 0x07); - reg0 = __lasx_xvsrli_b(tmp0, 5); - reg1 = __lasx_xvslli_b(reg1, 3); - tmpg = __lasx_xvor_v(reg1, reg0); - reg0 = __lasx_xvslli_b(tmpb, 3); - reg1 = __lasx_xvsrli_b(tmpb, 2); - tmpb = __lasx_xvor_v(reg1, reg0); - reg0 = __lasx_xvslli_b(tmpg, 2); - reg1 = __lasx_xvsrli_b(tmpg, 4); - tmpg = __lasx_xvor_v(reg1, reg0); - reg0 = __lasx_xvsrli_b(tmpr, 5); - tmpr = __lasx_xvor_v(tmpr, reg0); - DUP2_ARG2(__lasx_xvilvl_b, tmpg, tmpb, alpha, tmpr, reg0, reg1); - dst0 = __lasx_xvilvl_h(reg1, reg0); - dst1 = __lasx_xvilvh_h(reg1, reg0); - DUP2_ARG2(__lasx_xvilvh_b, tmpg, tmpb, alpha, tmpr, reg0, reg1); - dst2 = __lasx_xvilvl_h(reg1, reg0); - dst3 = __lasx_xvilvh_h(reg1, reg0); - DUP4_ARG3(__lasx_xvpermi_q, dst1, dst0, 0x20, dst1, dst0, 0x31, dst3, dst2, - 0x20, dst3, dst2, 0x31, reg0, reg1, reg2, reg3); - __lasx_xvst(reg0, dst_argb, 0); - __lasx_xvst(reg1, dst_argb, 32); - __lasx_xvst(reg2, dst_argb, 64); - __lasx_xvst(reg3, dst_argb, 96); - src_rgb565 += 64; - dst_argb += 128; - } -} - -void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 32; - __m256i src0, src1, src2; - __m256i tmp0, tmp1, tmp2; - __m256i dst0, dst1, dst2, dst3; - __m256i reg0, reg1, reg2, reg3; - __m256i alpha = __lasx_xvldi(0xFF); - __m256i shuf0 = {0x131211100F0E0D0C, 0x1B1A191817161514, 0x131211100F0E0D0C, - 0x1B1A191817161514}; - __m256i shuf1 = {0x1F1E1D1C1B1A1918, 0x0706050403020100, 0x1F1E1D1C1B1A1918, - 0x0706050403020100}; - __m256i shuf2 = {0x0B0A090807060504, 0x131211100F0E0D0C, 0x0B0A090807060504, - 0x131211100F0E0D0C}; - __m256i shuf3 = {0x1005040310020100, 0x100B0A0910080706, 0x1005040310020100, - 0x100B0A0910080706}; - - for (x = 0; x < len; x++) { - reg0 = __lasx_xvld(src_rgb24, 0); - reg1 = __lasx_xvld(src_rgb24, 32); - reg2 = __lasx_xvld(src_rgb24, 64); - src0 = __lasx_xvpermi_q(reg1, reg0, 0x30); - src1 = __lasx_xvpermi_q(reg2, reg0, 0x21); - src2 = __lasx_xvpermi_q(reg2, reg1, 0x30); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuf0, src1, src2, shuf1, tmp0, - tmp1); - tmp2 = __lasx_xvshuf_b(src1, src2, shuf2); - DUP4_ARG3(__lasx_xvshuf_b, alpha, src0, shuf3, alpha, tmp0, shuf3, alpha, - tmp1, shuf3, alpha, tmp2, shuf3, reg0, reg1, reg2, reg3); - DUP4_ARG3(__lasx_xvpermi_q, reg1, reg0, 0x20, reg3, reg2, 0x20, reg1, reg0, - 0x31, reg3, reg2, 0x31, dst0, dst1, dst2, dst3); - __lasx_xvst(dst0, dst_argb, 0); - __lasx_xvst(dst1, dst_argb, 32); - __lasx_xvst(dst2, dst_argb, 64); - __lasx_xvst(dst3, dst_argb, 96); - src_rgb24 += 96; - dst_argb += 128; - } -} - -void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { - int x; - int len = width / 32; - __m256i src0, src1, src2; - __m256i tmp0, tmp1, tmp2, reg0, reg1, reg2, reg3; - __m256i dst0, dst1, dst2, dst3; - __m256i alpha = __lasx_xvldi(0xFF); - __m256i shuf0 = {0x131211100F0E0D0C, 0x1B1A191817161514, 0x131211100F0E0D0C, - 0x1B1A191817161514}; - __m256i shuf1 = {0x1F1E1D1C1B1A1918, 0x0706050403020100, 0x1F1E1D1C1B1A1918, - 0x0706050403020100}; - __m256i shuf2 = {0x0B0A090807060504, 0x131211100F0E0D0C, 0x0B0A090807060504, - 0x131211100F0E0D0C}; - __m256i shuf3 = {0x1003040510000102, 0x10090A0B10060708, 0x1003040510000102, - 0x10090A0B10060708}; - - for (x = 0; x < len; x++) { - reg0 = __lasx_xvld(src_raw, 0); - reg1 = __lasx_xvld(src_raw, 32); - reg2 = __lasx_xvld(src_raw, 64); - src0 = __lasx_xvpermi_q(reg1, reg0, 0x30); - src1 = __lasx_xvpermi_q(reg2, reg0, 0x21); - src2 = __lasx_xvpermi_q(reg2, reg1, 0x30); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuf0, src1, src2, shuf1, tmp0, - tmp1); - tmp2 = __lasx_xvshuf_b(src1, src2, shuf2); - DUP4_ARG3(__lasx_xvshuf_b, alpha, src0, shuf3, alpha, tmp0, shuf3, alpha, - tmp1, shuf3, alpha, tmp2, shuf3, reg0, reg1, reg2, reg3); - DUP4_ARG3(__lasx_xvpermi_q, reg1, reg0, 0x20, reg3, reg2, 0x20, reg1, reg0, - 0x31, reg3, reg2, 0x31, dst0, dst1, dst2, dst3); - __lasx_xvst(dst0, dst_argb, 0); - __lasx_xvst(dst1, dst_argb, 32); - __lasx_xvst(dst2, dst_argb, 64); - __lasx_xvst(dst3, dst_argb, 96); - src_raw += 96; - dst_argb += 128; - } -} - -void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { - int x; - int len = width / 32; - __m256i reg0, reg1, reg2; - __m256i src0, src1, src2; - __m256i tmp0, tmp1, tmp2; - __m256i dst0, dst1, dst2; - __m256i shuf0 = {0x0708030405000102, 0x110C0D0E090A0B06, 0x0708030405000102, - 0x110C0D0E090A0B06}; - __m256i shuf1 = {0x1516171213140F10, 0x1F1E1B1C1D18191A, 0x1516171213140F10, - 0x1F1E1B1C1D18191A}; - __m256i shuf2 = {0x090405060102031E, 0x0D0E0F0A0B0C0708, 0x090405060102031E, - 0x0D0E0F0A0B0C0708}; - - for (x = 0; x < len; x++) { - reg0 = __lasx_xvld(src_raw, 0); - reg1 = __lasx_xvld(src_raw, 32); - reg2 = __lasx_xvld(src_raw, 64); - src0 = __lasx_xvpermi_q(reg1, reg0, 0x30); - src1 = __lasx_xvpermi_q(reg2, reg0, 0x21); - src2 = __lasx_xvpermi_q(reg2, reg1, 0x30); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuf0, src1, src0, shuf1, tmp0, - tmp1); - tmp2 = __lasx_xvshuf_b(src1, src2, shuf2); - tmp1 = __lasx_xvextrins_b(tmp1, src2, 0xE0); - dst0 = __lasx_xvpermi_q(tmp1, tmp0, 0x20); - dst1 = __lasx_xvpermi_q(tmp0, tmp2, 0x30); - dst2 = __lasx_xvpermi_q(tmp2, tmp1, 0x31); - __lasx_xvst(dst0, dst_rgb24, 0); - __lasx_xvst(dst1, dst_rgb24, 32); - __lasx_xvst(dst2, dst_rgb24, 64); - src_raw += 96; - dst_rgb24 += 96; - } -} - -void ARGB1555ToYRow_LASX(const uint8_t* src_argb1555, - uint8_t* dst_y, - int width) { - int x; - int len = width / 32; - __m256i src0, src1; - __m256i tmp0, tmp1, tmpb, tmpg, tmpr; - __m256i reg0, reg1, reg2, dst0; - __m256i const_66 = __lasx_xvldi(66); - __m256i const_129 = __lasx_xvldi(129); - __m256i const_25 = __lasx_xvldi(25); - __m256i const_1080 = {0x1080108010801080, 0x1080108010801080, - 0x1080108010801080, 0x1080108010801080}; - - for (x = 0; x < len; x++) { - src0 = __lasx_xvld(src_argb1555, 0); - src1 = __lasx_xvld(src_argb1555, 32); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - tmpb = __lasx_xvandi_b(tmp0, 0x1F); - tmpg = __lasx_xvsrli_b(tmp0, 5); - reg0 = __lasx_xvandi_b(tmp1, 0x03); - reg0 = __lasx_xvslli_b(reg0, 3); - tmpg = __lasx_xvor_v(tmpg, reg0); - reg1 = __lasx_xvandi_b(tmp1, 0x7C); - tmpr = __lasx_xvsrli_b(reg1, 2); - reg0 = __lasx_xvslli_b(tmpb, 3); - reg1 = __lasx_xvslli_b(tmpg, 3); - reg2 = __lasx_xvslli_b(tmpr, 3); - tmpb = __lasx_xvsrli_b(tmpb, 2); - tmpg = __lasx_xvsrli_b(tmpg, 2); - tmpr = __lasx_xvsrli_b(tmpr, 2); - tmpb = __lasx_xvor_v(reg0, tmpb); - tmpg = __lasx_xvor_v(reg1, tmpg); - tmpr = __lasx_xvor_v(reg2, tmpr); - reg0 = __lasx_xvmaddwev_h_bu(const_1080, tmpb, const_25); - reg1 = __lasx_xvmaddwod_h_bu(const_1080, tmpb, const_25); - reg0 = __lasx_xvmaddwev_h_bu(reg0, tmpg, const_129); - reg1 = __lasx_xvmaddwod_h_bu(reg1, tmpg, const_129); - reg0 = __lasx_xvmaddwev_h_bu(reg0, tmpr, const_66); - reg1 = __lasx_xvmaddwod_h_bu(reg1, tmpr, const_66); - dst0 = __lasx_xvpackod_b(reg1, reg0); - dst0 = __lasx_xvpermi_d(dst0, 0xD8); - __lasx_xvst(dst0, dst_y, 0); - src_argb1555 += 64; - dst_y += 32; - } -} - -void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, - int src_stride_argb1555, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 32; - const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555; - __m256i src0, src1, src2, src3; - __m256i tmp0, tmp1, tmp2, tmp3; - __m256i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m256i reg0, reg1, reg2, reg3, dst0; - __m256i const_112 = __lasx_xvldi(0x470); - __m256i const_74 = __lasx_xvldi(0x44A); - __m256i const_38 = __lasx_xvldi(0x426); - __m256i const_94 = __lasx_xvldi(0x45E); - __m256i const_18 = __lasx_xvldi(0x412); - __m256i const_8000 = (__m256i)v4u64{0x8000800080008000, 0x8000800080008000, - 0x8000800080008000, 0x8000800080008000}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lasx_xvld, src_argb1555, 0, src_argb1555, 32, next_argb1555, 0, - next_argb1555, 32, src0, src1, src2, src3); - DUP2_ARG2(__lasx_xvpickev_b, src1, src0, src3, src2, tmp0, tmp2); - DUP2_ARG2(__lasx_xvpickod_b, src1, src0, src3, src2, tmp1, tmp3); - tmpb = __lasx_xvandi_b(tmp0, 0x1F); - nexb = __lasx_xvandi_b(tmp2, 0x1F); - tmpg = __lasx_xvsrli_b(tmp0, 5); - nexg = __lasx_xvsrli_b(tmp2, 5); - reg0 = __lasx_xvandi_b(tmp1, 0x03); - reg2 = __lasx_xvandi_b(tmp3, 0x03); - reg0 = __lasx_xvslli_b(reg0, 3); - reg2 = __lasx_xvslli_b(reg2, 3); - tmpg = __lasx_xvor_v(tmpg, reg0); - nexg = __lasx_xvor_v(nexg, reg2); - reg1 = __lasx_xvandi_b(tmp1, 0x7C); - reg3 = __lasx_xvandi_b(tmp3, 0x7C); - tmpr = __lasx_xvsrli_b(reg1, 2); - nexr = __lasx_xvsrli_b(reg3, 2); - reg0 = __lasx_xvslli_b(tmpb, 3); - reg1 = __lasx_xvslli_b(tmpg, 3); - reg2 = __lasx_xvslli_b(tmpr, 3); - tmpb = __lasx_xvsrli_b(tmpb, 2); - tmpg = __lasx_xvsrli_b(tmpg, 2); - tmpr = __lasx_xvsrli_b(tmpr, 2); - tmpb = __lasx_xvor_v(reg0, tmpb); - tmpg = __lasx_xvor_v(reg1, tmpg); - tmpr = __lasx_xvor_v(reg2, tmpr); - reg0 = __lasx_xvslli_b(nexb, 3); - reg1 = __lasx_xvslli_b(nexg, 3); - reg2 = __lasx_xvslli_b(nexr, 3); - nexb = __lasx_xvsrli_b(nexb, 2); - nexg = __lasx_xvsrli_b(nexg, 2); - nexr = __lasx_xvsrli_b(nexr, 2); - nexb = __lasx_xvor_v(reg0, nexb); - nexg = __lasx_xvor_v(reg1, nexg); - nexr = __lasx_xvor_v(reg2, nexr); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, reg0, reg1); - reg0 = __lasx_xvpermi_d(reg0, 0xD8); - reg1 = __lasx_xvpermi_d(reg1, 0xD8); - dst0 = __lasx_xvpickod_b(reg1, reg0); - __lasx_xvstelm_d(dst0, dst_u, 0, 0); - __lasx_xvstelm_d(dst0, dst_v, 0, 1); - __lasx_xvstelm_d(dst0, dst_u, 8, 2); - __lasx_xvstelm_d(dst0, dst_v, 8, 3); - src_argb1555 += 64; - next_argb1555 += 64; - dst_u += 16; - dst_v += 16; - } -} - -void RGB565ToYRow_LASX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { - int x; - int len = width / 32; - __m256i src0, src1; - __m256i tmp0, tmp1, tmpb, tmpg, tmpr; - __m256i reg0, reg1, dst0; - __m256i const_66 = __lasx_xvldi(66); - __m256i const_129 = __lasx_xvldi(129); - __m256i const_25 = __lasx_xvldi(25); - __m256i const_1080 = {0x1080108010801080, 0x1080108010801080, - 0x1080108010801080, 0x1080108010801080}; - - for (x = 0; x < len; x++) { - src0 = __lasx_xvld(src_rgb565, 0); - src1 = __lasx_xvld(src_rgb565, 32); - tmp0 = __lasx_xvpickev_b(src1, src0); - tmp1 = __lasx_xvpickod_b(src1, src0); - tmpb = __lasx_xvandi_b(tmp0, 0x1F); - tmpr = __lasx_xvandi_b(tmp1, 0xF8); - reg1 = __lasx_xvandi_b(tmp1, 0x07); - reg0 = __lasx_xvsrli_b(tmp0, 5); - reg1 = __lasx_xvslli_b(reg1, 3); - tmpg = __lasx_xvor_v(reg1, reg0); - reg0 = __lasx_xvslli_b(tmpb, 3); - reg1 = __lasx_xvsrli_b(tmpb, 2); - tmpb = __lasx_xvor_v(reg1, reg0); - reg0 = __lasx_xvslli_b(tmpg, 2); - reg1 = __lasx_xvsrli_b(tmpg, 4); - tmpg = __lasx_xvor_v(reg1, reg0); - reg0 = __lasx_xvsrli_b(tmpr, 5); - tmpr = __lasx_xvor_v(tmpr, reg0); - reg0 = __lasx_xvmaddwev_h_bu(const_1080, tmpb, const_25); - reg1 = __lasx_xvmaddwod_h_bu(const_1080, tmpb, const_25); - reg0 = __lasx_xvmaddwev_h_bu(reg0, tmpg, const_129); - reg1 = __lasx_xvmaddwod_h_bu(reg1, tmpg, const_129); - reg0 = __lasx_xvmaddwev_h_bu(reg0, tmpr, const_66); - reg1 = __lasx_xvmaddwod_h_bu(reg1, tmpr, const_66); - dst0 = __lasx_xvpackod_b(reg1, reg0); - dst0 = __lasx_xvpermi_d(dst0, 0xD8); - __lasx_xvst(dst0, dst_y, 0); - dst_y += 32; - src_rgb565 += 64; - } -} - -void RGB565ToUVRow_LASX(const uint8_t* src_rgb565, - int src_stride_rgb565, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 32; - const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565; - __m256i src0, src1, src2, src3; - __m256i tmp0, tmp1, tmp2, tmp3; - __m256i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m256i reg0, reg1, reg2, reg3, dst0; - __m256i const_112 = __lasx_xvldi(0x470); - __m256i const_74 = __lasx_xvldi(0x44A); - __m256i const_38 = __lasx_xvldi(0x426); - __m256i const_94 = __lasx_xvldi(0x45E); - __m256i const_18 = __lasx_xvldi(0x412); - __m256i const_8000 = (__m256i)v4u64{0x8000800080008000, 0x8000800080008000, - 0x8000800080008000, 0x8000800080008000}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lasx_xvld, src_rgb565, 0, src_rgb565, 32, next_rgb565, 0, - next_rgb565, 32, src0, src1, src2, src3); - DUP2_ARG2(__lasx_xvpickev_b, src1, src0, src3, src2, tmp0, tmp2); - DUP2_ARG2(__lasx_xvpickod_b, src1, src0, src3, src2, tmp1, tmp3); - tmpb = __lasx_xvandi_b(tmp0, 0x1F); - tmpr = __lasx_xvandi_b(tmp1, 0xF8); - nexb = __lasx_xvandi_b(tmp2, 0x1F); - nexr = __lasx_xvandi_b(tmp3, 0xF8); - reg1 = __lasx_xvandi_b(tmp1, 0x07); - reg3 = __lasx_xvandi_b(tmp3, 0x07); - reg0 = __lasx_xvsrli_b(tmp0, 5); - reg1 = __lasx_xvslli_b(reg1, 3); - reg2 = __lasx_xvsrli_b(tmp2, 5); - reg3 = __lasx_xvslli_b(reg3, 3); - tmpg = __lasx_xvor_v(reg1, reg0); - nexg = __lasx_xvor_v(reg2, reg3); - reg0 = __lasx_xvslli_b(tmpb, 3); - reg1 = __lasx_xvsrli_b(tmpb, 2); - reg2 = __lasx_xvslli_b(nexb, 3); - reg3 = __lasx_xvsrli_b(nexb, 2); - tmpb = __lasx_xvor_v(reg1, reg0); - nexb = __lasx_xvor_v(reg2, reg3); - reg0 = __lasx_xvslli_b(tmpg, 2); - reg1 = __lasx_xvsrli_b(tmpg, 4); - reg2 = __lasx_xvslli_b(nexg, 2); - reg3 = __lasx_xvsrli_b(nexg, 4); - tmpg = __lasx_xvor_v(reg1, reg0); - nexg = __lasx_xvor_v(reg2, reg3); - reg0 = __lasx_xvsrli_b(tmpr, 5); - reg2 = __lasx_xvsrli_b(nexr, 5); - tmpr = __lasx_xvor_v(tmpr, reg0); - nexr = __lasx_xvor_v(nexr, reg2); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, reg0, reg1); - reg0 = __lasx_xvpermi_d(reg0, 0xD8); - reg1 = __lasx_xvpermi_d(reg1, 0xD8); - dst0 = __lasx_xvpickod_b(reg1, reg0); - __lasx_xvstelm_d(dst0, dst_u, 0, 0); - __lasx_xvstelm_d(dst0, dst_v, 0, 1); - __lasx_xvstelm_d(dst0, dst_u, 8, 2); - __lasx_xvstelm_d(dst0, dst_v, 8, 3); - dst_u += 16; - dst_v += 16; - src_rgb565 += 64; - next_rgb565 += 64; - } -} - -void RGB24ToUVRow_LASX(const uint8_t* src_rgb24, - int src_stride_rgb24, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - const uint8_t* next_rgb24 = src_rgb24 + src_stride_rgb24; - int len = width / 32; - __m256i src0, src1, src2, reg0, reg1, reg2; - __m256i nex0, nex1, nex2, dst0, tmp0, tmp1, tmp2; - __m256i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m256i const_112 = __lasx_xvldi(0x470); - __m256i const_74 = __lasx_xvldi(0x44A); - __m256i const_38 = __lasx_xvldi(0x426); - __m256i const_94 = __lasx_xvldi(0x45E); - __m256i const_18 = __lasx_xvldi(0x412); - __m256i const_8000 = (__m256i)v4u64{0x8000800080008000, 0x8000800080008000, - 0x8000800080008000, 0x8000800080008000}; - __m256i shuff0_b = {0x15120F0C09060300, 0x00000000001E1B18, - 0x15120F0C09060300, 0x00000000001E1B18}; - __m256i shuff1_b = {0x0706050403020100, 0x1D1A1714110A0908, - 0x0706050403020100, 0x1D1A1714110A0908}; - __m256i shuff0_g = {0x1613100D0A070401, 0x00000000001F1C19, - 0x1613100D0A070401, 0x00000000001F1C19}; - __m256i shuff1_g = {0x0706050403020100, 0x1E1B1815120A0908, - 0x0706050403020100, 0x1E1B1815120A0908}; - __m256i shuff0_r = {0x1714110E0B080502, 0x0000000000001D1A, - 0x1714110E0B080502, 0x0000000000001D1A}; - __m256i shuff1_r = {0x0706050403020100, 0x1F1C191613100908, - 0x0706050403020100, 0x1F1C191613100908}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lasx_xvld, src_rgb24, 0, src_rgb24, 32, src_rgb24, 64, - next_rgb24, 0, reg0, reg1, reg2, tmp0); - DUP2_ARG2(__lasx_xvld, next_rgb24, 32, next_rgb24, 64, tmp1, tmp2); - DUP4_ARG3(__lasx_xvpermi_q, reg1, reg0, 0x30, reg2, reg0, 0x21, reg2, reg1, - 0x30, tmp1, tmp0, 0x30, src0, src1, src2, nex0); - DUP2_ARG3(__lasx_xvpermi_q, tmp2, tmp0, 0x21, tmp2, tmp1, 0x30, nex1, nex2); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuff0_b, nex1, nex0, shuff0_b, tmpb, - nexb); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuff0_g, nex1, nex0, shuff0_g, tmpg, - nexg); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuff0_r, nex1, nex0, shuff0_r, tmpr, - nexr); - DUP2_ARG3(__lasx_xvshuf_b, src2, tmpb, shuff1_b, nex2, nexb, shuff1_b, tmpb, - nexb); - DUP2_ARG3(__lasx_xvshuf_b, src2, tmpg, shuff1_g, nex2, nexg, shuff1_g, tmpg, - nexg); - DUP2_ARG3(__lasx_xvshuf_b, src2, tmpr, shuff1_r, nex2, nexr, shuff1_r, tmpr, - nexr); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, reg0, reg1); - dst0 = __lasx_xvpickod_b(reg1, reg0); - __lasx_xvstelm_d(dst0, dst_u, 0, 0); - __lasx_xvstelm_d(dst0, dst_v, 0, 1); - __lasx_xvstelm_d(dst0, dst_u, 8, 2); - __lasx_xvstelm_d(dst0, dst_v, 8, 3); - src_rgb24 += 96; - next_rgb24 += 96; - dst_u += 16; - dst_v += 16; - } -} - -void RAWToUVRow_LASX(const uint8_t* src_raw, - int src_stride_raw, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - const uint8_t* next_raw = src_raw + src_stride_raw; - int len = width / 32; - __m256i src0, src1, src2, reg0, reg1, reg2; - __m256i nex0, nex1, nex2, dst0, tmp0, tmp1, tmp2; - __m256i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m256i const_112 = __lasx_xvldi(0x470); - __m256i const_74 = __lasx_xvldi(0x44A); - __m256i const_38 = __lasx_xvldi(0x426); - __m256i const_94 = __lasx_xvldi(0x45E); - __m256i const_18 = __lasx_xvldi(0x412); - __m256i const_8000 = (__m256i)v4u64{0x8000800080008000, 0x8000800080008000, - 0x8000800080008000, 0x8000800080008000}; - __m256i shuff0_r = {0x15120F0C09060300, 0x00000000001E1B18, - 0x15120F0C09060300, 0x00000000001E1B18}; - __m256i shuff1_r = {0x0706050403020100, 0x1D1A1714110A0908, - 0x0706050403020100, 0x1D1A1714110A0908}; - __m256i shuff0_g = {0x1613100D0A070401, 0x00000000001F1C19, - 0x1613100D0A070401, 0x00000000001F1C19}; - __m256i shuff1_g = {0x0706050403020100, 0x1E1B1815120A0908, - 0x0706050403020100, 0x1E1B1815120A0908}; - __m256i shuff0_b = {0x1714110E0B080502, 0x0000000000001D1A, - 0x1714110E0B080502, 0x0000000000001D1A}; - __m256i shuff1_b = {0x0706050403020100, 0x1F1C191613100908, - 0x0706050403020100, 0x1F1C191613100908}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lasx_xvld, src_raw, 0, src_raw, 32, src_raw, 64, next_raw, 0, - reg0, reg1, reg2, tmp0); - DUP2_ARG2(__lasx_xvld, next_raw, 32, next_raw, 64, tmp1, tmp2); - DUP4_ARG3(__lasx_xvpermi_q, reg1, reg0, 0x30, reg2, reg0, 0x21, reg2, reg1, - 0x30, tmp1, tmp0, 0x30, src0, src1, src2, nex0); - DUP2_ARG3(__lasx_xvpermi_q, tmp2, tmp0, 0x21, tmp2, tmp1, 0x30, nex1, nex2); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuff0_b, nex1, nex0, shuff0_b, tmpb, - nexb); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuff0_g, nex1, nex0, shuff0_g, tmpg, - nexg); - DUP2_ARG3(__lasx_xvshuf_b, src1, src0, shuff0_r, nex1, nex0, shuff0_r, tmpr, - nexr); - DUP2_ARG3(__lasx_xvshuf_b, src2, tmpb, shuff1_b, nex2, nexb, shuff1_b, tmpb, - nexb); - DUP2_ARG3(__lasx_xvshuf_b, src2, tmpg, shuff1_g, nex2, nexg, shuff1_g, tmpg, - nexg); - DUP2_ARG3(__lasx_xvshuf_b, src2, tmpr, shuff1_r, nex2, nexr, shuff1_r, tmpr, - nexr); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, reg0, reg1); - dst0 = __lasx_xvpickod_b(reg1, reg0); - __lasx_xvstelm_d(dst0, dst_u, 0, 0); - __lasx_xvstelm_d(dst0, dst_v, 0, 1); - __lasx_xvstelm_d(dst0, dst_u, 8, 2); - __lasx_xvstelm_d(dst0, dst_v, 8, 3); - src_raw += 96; - next_raw += 96; - dst_u += 16; - dst_v += 16; - } + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr14, 31\n\t" + "xvsrli.h $xr14, $xr14, 8\n\t" + "xvldi $xr15, 128\n\t" + "xvsrli.h $xr15, $xr15, 8\n\t" + "xvslli.h $xr15, $xr15, 8\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvldrepl.d $xr10, %[su], 0\n\t" + "xvshuf4i.w $xr31, $xr10, 0x01\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "xvldrepl.d $xr11, %[sv], 0\n\t" + "xvshuf4i.w $xr31, $xr11, 0x01\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvilvl.b $xr10, $xr10, $xr10\n\t" + "xvilvl.b $xr11, $xr11, $xr11\n\t" + "xvilvl.b $xr1, $xr11, $xr10\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvsrli.b $xr2, $xr2, 3\n\t" + "xvsrli.b $xr3, $xr3, 3\n\t" + "xvsrli.b $xr4, $xr4, 3\n\t" + "xvilvl.b $xr2, $xr2, $xr2\n\t" + "xvilvl.b $xr3, $xr3, $xr3\n\t" + "xvilvl.b $xr4, $xr4, $xr4\n\t" + "xvand.v $xr2, $xr2, $xr14\n\t" + "xvand.v $xr3, $xr3, $xr14\n\t" + "xvand.v $xr4, $xr4, $xr14\n\t" + "xvslli.h $xr3, $xr3, 5\n\t" + "xvslli.h $xr4, $xr4, 10\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvor.v $xr2, $xr2, $xr4\n\t" + "xvor.v $xr2, $xr2, $xr15\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr10", "$xr11", + "$xr14", "$xr15", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr31"); } void NV12ToARGBRow_LASX(const uint8_t* src_y, @@ -1844,217 +961,2166 @@ void NV12ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m256i vec_yg, vec_yb, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_vrub, vec_vgug, vec_y, vec_vu; - __m256i out_b, out_g, out_r; - __m256i const_0x80 = __lasx_xvldi(0x80); - __m256i alpha = __lasx_xvldi(0xFF); + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "vld $vr1, %[suv], 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvilvl.h $xr1, $xr1, $xr1\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr3, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr4\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[suv], %[suv], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", + "$xr31"); +} - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_vrub = __lasx_xvilvl_h(vec_vr, vec_ub); - vec_vgug = __lasx_xvilvl_h(vec_vg, vec_ug); - - for (x = 0; x < len; x++) { - vec_y = __lasx_xvld(src_y, 0); - vec_vu = __lasx_xvld(src_uv, 0); - vec_vu = __lasx_xvsub_b(vec_vu, const_0x80); - vec_vu = __lasx_vext2xv_h_b(vec_vu); - YUVTORGB(vec_y, vec_vu, vec_vrub, vec_vgug, vec_yg, vec_yb, out_r, out_g, - out_b); - STOREARGB(alpha, out_r, out_g, out_b, dst_argb); - src_y += 16; - src_uv += 16; - } +void NV21ToARGBRow_LASX(const uint8_t* src_y, + const uint8_t* src_vu, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "vld $vr1, %[suv], 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvshuf4i.b $xr1, $xr1, 0xb1\n\t" + "xvilvl.h $xr1, $xr1, $xr1\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr3, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr4\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[suv], %[suv], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_vu), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", + "$xr31"); } void NV12ToRGB565Row_LASX(const uint8_t* src_y, const uint8_t* src_uv, - uint8_t* dst_rgb565, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m256i vec_yg, vec_yb, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_vrub, vec_vgug, vec_y, vec_vu; - __m256i out_b, out_g, out_r; - __m256i const_0x80 = __lasx_xvldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_vrub = __lasx_xvilvl_h(vec_vr, vec_ub); - vec_vgug = __lasx_xvilvl_h(vec_vg, vec_ug); - - for (x = 0; x < len; x++) { - vec_y = __lasx_xvld(src_y, 0); - vec_vu = __lasx_xvld(src_uv, 0); - vec_vu = __lasx_xvsub_b(vec_vu, const_0x80); - vec_vu = __lasx_vext2xv_h_b(vec_vu); - YUVTORGB(vec_y, vec_vu, vec_vrub, vec_vgug, vec_yg, vec_yb, out_r, out_g, - out_b); - out_b = __lasx_xvsrli_h(out_b, 3); - out_g = __lasx_xvsrli_h(out_g, 2); - out_r = __lasx_xvsrli_h(out_r, 3); - out_g = __lasx_xvslli_h(out_g, 5); - out_r = __lasx_xvslli_h(out_r, 11); - out_r = __lasx_xvor_v(out_r, out_g); - out_r = __lasx_xvor_v(out_r, out_b); - __lasx_xvst(out_r, dst_rgb565, 0); - src_y += 16; - src_uv += 16; - dst_rgb565 += 32; - } + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr14, 31\n\t" + "xvsrli.h $xr14, $xr14, 8\n\t" + "xvldi $xr15, 63\n\t" + "xvsrli.h $xr15, $xr15, 8\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "vld $vr1, %[suv], 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvilvl.h $xr1, $xr1, $xr1\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvsrli.b $xr2, $xr2, 3\n\t" + "xvsrli.b $xr3, $xr3, 2\n\t" + "xvsrli.b $xr4, $xr4, 3\n\t" + "xvilvl.b $xr2, $xr2, $xr2\n\t" + "xvilvl.b $xr3, $xr3, $xr3\n\t" + "xvilvl.b $xr4, $xr4, $xr4\n\t" + "xvand.v $xr2, $xr2, $xr14\n\t" + "xvand.v $xr3, $xr3, $xr15\n\t" + "xvand.v $xr4, $xr4, $xr14\n\t" + "xvslli.h $xr3, $xr3, 5\n\t" + "xvslli.h $xr4, $xr4, 11\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvor.v $xr2, $xr2, $xr4\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[suv], %[suv], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr14", "$xr15", + "$xr20", "$xr21", "$xr22", "$xr23", "$xr24"); } -void NV21ToARGBRow_LASX(const uint8_t* src_y, - const uint8_t* src_uv, +void I444ToARGBRow_LASX(const uint8_t* src_y, + const uint8_t* src_u, + const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m256i vec_yg, vec_yb, vec_ub, vec_vr, vec_ug, vec_vg; - __m256i vec_ubvr, vec_ugvg, vec_y, vec_uv; - __m256i out_b, out_g, out_r; - __m256i const_0x80 = __lasx_xvldi(0x80); - __m256i alpha = __lasx_xvldi(0xFF); - - YUVTORGB_SETUP(yuvconstants, vec_ub, vec_vr, vec_ug, vec_vg, vec_yg, vec_yb); - vec_ubvr = __lasx_xvilvl_h(vec_ub, vec_vr); - vec_ugvg = __lasx_xvilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - vec_y = __lasx_xvld(src_y, 0); - vec_uv = __lasx_xvld(src_uv, 0); - vec_uv = __lasx_xvsub_b(vec_uv, const_0x80); - vec_uv = __lasx_vext2xv_h_b(vec_uv); - YUVTORGB(vec_y, vec_uv, vec_ubvr, vec_ugvg, vec_yg, vec_yb, out_b, out_g, - out_r); - STOREARGB(alpha, out_r, out_g, out_b, dst_argb); - src_y += 16; - src_uv += 16; - } + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "vld $vr10, %[su], 0\n\t" + "xvpermi.d $xr10, $xr10, 0xd8\n\t" + "vld $vr11, %[sv], 0\n\t" + "xvpermi.d $xr11, $xr11, 0xd8\n\t" + "xvilvl.b $xr1, $xr11, $xr10\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr3, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr4\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 16\n\t" + "addi.d %[sv], %[sv], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr20", "$xr21", "$xr22", + "$xr23", "$xr24", "$xr31"); +} + +void I400ToARGBRow_LASX(const uint8_t* src_y, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "vld $vr23, %[c], 96\n\t" + "xvpermi.q $xr23, $xr23, 0x00\n\t" + "vld $vr24, %[c], 128\n\t" + "xvpermi.q $xr24, $xr24, 0x00\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvssrani.bu.h $xr2, $xr0, 6\n\t" + "xvilvl.b $xr6, $xr2, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr2\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr2", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", + "$xr23", "$xr24", "$xr31"); +} + +void J400ToARGBRow_LASX(const uint8_t* src_y, uint8_t* dst_argb, int width) { + asm volatile( + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "vld $vr2, %[sy], 0\n\t" + "xvpermi.d $xr2, $xr2, 0xd8\n\t" + "xvilvl.b $xr6, $xr2, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr2\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr2", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr31"); +} + +void YUY2ToARGBRow_LASX(const uint8_t* src_yuy2, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "xvld $xr10, %[src], 0\n\t" + "xvpickev.b $xr0, $xr10, $xr10\n\t" + "xvpickod.b $xr1, $xr10, $xr10\n\t" + "xvilvl.h $xr1, $xr1, $xr1\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr3, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr4\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr20", "$xr21", "$xr22", "$xr23", + "$xr24", "$xr31"); +} + +void UYVYToARGBRow_LASX(const uint8_t* src_uyvy, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, + int width) { + asm volatile( + "xvldrepl.w $xr20, %[c], 0\n\t" + "xvldrepl.w $xr21, %[c], 32\n\t" + "xvldrepl.w $xr22, %[c], 64\n\t" + "xvldrepl.h $xr23, %[c], 96\n\t" + "xvldrepl.h $xr24, %[c], 128\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "xvld $xr10, %[src], 0\n\t" + "xvpickod.b $xr0, $xr10, $xr10\n\t" + "xvpickev.b $xr1, $xr10, $xr10\n\t" + "xvilvl.h $xr1, $xr1, $xr1\n\t" + "xvilvl.b $xr0, $xr0, $xr0\n\t" + "xvxori.b $xr1, $xr1, 0x80\n\t" + "xvmuh.hu $xr0, $xr0, $xr23\n\t" + "xvadd.h $xr0, $xr0, $xr24\n\t" + "xvmulwev.h.bu.b $xr2, $xr20, $xr1\n\t" + "xvmulwod.h.bu.b $xr4, $xr22, $xr1\n\t" + "xvmulwev.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvmaddwod.h.bu.b $xr3, $xr21, $xr1\n\t" + "xvsadd.h $xr2, $xr0, $xr2\n\t" + "xvsadd.h $xr4, $xr0, $xr4\n\t" + "xvssub.h $xr3, $xr0, $xr3\n\t" + "xvssrani.bu.h $xr2, $xr2, 6\n\t" + "xvssrani.bu.h $xr3, $xr3, 6\n\t" + "xvssrani.bu.h $xr4, $xr4, 6\n\t" + "xvilvl.b $xr6, $xr3, $xr2\n\t" + "xvilvl.b $xr7, $xr5, $xr4\n\t" + "xvilvl.h $xr8, $xr7, $xr6\n\t" + "xvilvh.h $xr9, $xr7, $xr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr8, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr9, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr9, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr20", "$xr21", "$xr22", "$xr23", + "$xr24", "$xr31"); +} + +static LIBYUV_NOINLINE void ARGBTo24Helper_LASX(const uint8_t* src_argb, + uint8_t* dst_24, + const uint8_t* shuf, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 64\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 80\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "vld $vr2, %[src], 32\n\t" + "vld $vr31, %[src], 96\n\t" + "xvpermi.q $xr2, $xr31, 0x02\n\t" + "vld $vr3, %[src], 48\n\t" + "vld $vr31, %[src], 112\n\t" + "xvpermi.q $xr3, $xr31, 0x02\n\t" + "xvshuf.b $xr0, $xr0, $xr0, $xr4\n\t" + "xvshuf.b $xr1, $xr1, $xr1, $xr4\n\t" + "xvshuf.b $xr2, $xr2, $xr2, $xr4\n\t" + "xvshuf.b $xr3, $xr3, $xr3, $xr4\n\t" + "vstelm.d $vr0, %[dst], 0, 0\n\t" + "xvpermi.q $xr31, $xr0, 0x01\n\t" + "vstelm.d $vr31, %[dst], 48, 0\n\t" + "vstelm.w $vr0, %[dst], 8, 2\n\t" + "xvpermi.q $xr31, $xr0, 0x01\n\t" + "vstelm.w $vr31, %[dst], 56, 2\n\t" + "vstelm.w $vr1, %[dst], 12, 0\n\t" + "vstelm.w $vr1, %[dst], 16, 1\n\t" + "xvpermi.q $xr31, $xr1, 0x01\n\t" + "vstelm.w $vr31, %[dst], 60, 0\n\t" + "vstelm.w $vr31, %[dst], 64, 1\n\t" + "vstelm.w $vr1, %[dst], 20, 2\n\t" + "xvpermi.q $xr31, $xr1, 0x01\n\t" + "vstelm.w $vr31, %[dst], 68, 2\n\t" + "vstelm.d $vr2, %[dst], 24, 0\n\t" + "xvpermi.q $xr31, $xr2, 0x01\n\t" + "vstelm.d $vr31, %[dst], 72, 0\n\t" + "vstelm.w $vr2, %[dst], 32, 2\n\t" + "xvpermi.q $xr31, $xr2, 0x01\n\t" + "vstelm.w $vr31, %[dst], 80, 2\n\t" + "vstelm.w $vr3, %[dst], 36, 0\n\t" + "vstelm.w $vr3, %[dst], 40, 1\n\t" + "xvpermi.q $xr31, $xr3, 0x01\n\t" + "vstelm.w $vr31, %[dst], 84, 0\n\t" + "vstelm.w $vr31, %[dst], 88, 1\n\t" + "vstelm.w $vr3, %[dst], 44, 2\n\t" + "xvpermi.q $xr31, $xr3, 0x01\n\t" + "vstelm.w $vr31, %[dst], 92, 2\n\t" + "addi.d %[src], %[src], 128\n\t" + "addi.d %[dst], %[dst], 96\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_24), [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); +} + +void ARGBToRGB24Row_LASX(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width) { + static const uint8_t kShuf[16] = {0, 1, 2, 4, 5, 6, 8, 9, + 10, 12, 13, 14, 0, 0, 0, 0}; + ARGBTo24Helper_LASX(src_argb, dst_rgb24, kShuf, width); +} + +void ARGBToRAWRow_LASX(const uint8_t* src_argb, uint8_t* dst_raw, int width) { + static const uint8_t kShuf[16] = {2, 1, 0, 6, 5, 4, 10, 9, + 8, 14, 13, 12, 0, 0, 0, 0}; + ARGBTo24Helper_LASX(src_argb, dst_raw, kShuf, width); +} + +static LIBYUV_NOINLINE void RGB24ToARGBHelper_LASX(const uint8_t* src_24, + uint8_t* dst_argb, + const uint8_t* shuf, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "vld $vr5, %[shuf], 16\n\t" + "xvpermi.q $xr5, $xr5, 0x00\n\t" + "xvldi $xr6, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 12\n\t" + "vld $vr31, %[src], 60\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "vld $vr2, %[src], 24\n\t" + "vld $vr31, %[src], 72\n\t" + "xvpermi.q $xr2, $xr31, 0x02\n\t" + "vld $vr3, %[src], 32\n\t" + "vld $vr31, %[src], 80\n\t" + "xvpermi.q $xr3, $xr31, 0x02\n\t" + "xvshuf.b $xr0, $xr6, $xr0, $xr4\n\t" + "xvshuf.b $xr1, $xr6, $xr1, $xr4\n\t" + "xvshuf.b $xr2, $xr6, $xr2, $xr4\n\t" + "xvshuf.b $xr3, $xr6, $xr3, $xr5\n\t" + "vst $vr0, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr0, 0x01\n\t" + "vst $vr31, %[dst], 64\n\t" + "vst $vr1, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr1, 0x01\n\t" + "vst $vr31, %[dst], 80\n\t" + "vst $vr2, %[dst], 32\n\t" + "xvpermi.q $xr31, $xr2, 0x01\n\t" + "vst $vr31, %[dst], 96\n\t" + "vst $vr3, %[dst], 48\n\t" + "xvpermi.q $xr31, $xr3, 0x01\n\t" + "vst $vr31, %[dst], 112\n\t" + "addi.d %[src], %[src], 96\n\t" + "addi.d %[dst], %[dst], 128\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_24), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr31"); +} + +void RGB24ToARGBRow_LASX(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width) { + static const uint8_t kShuf[32] = {0, 1, 2, 16, 3, 4, 5, 16, 6, 7, 8, + 16, 9, 10, 11, 16, 4, 5, 6, 16, 7, 8, + 9, 16, 10, 11, 12, 16, 13, 14, 15, 16}; + RGB24ToARGBHelper_LASX(src_rgb24, dst_argb, kShuf, width); +} + +void RAWToARGBRow_LASX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { + static const uint8_t kShuf[32] = {2, 1, 0, 16, 5, 4, 3, 16, 8, 7, 6, + 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, 8, + 7, 16, 12, 11, 10, 16, 15, 14, 13, 16}; + RGB24ToARGBHelper_LASX(src_raw, dst_argb, kShuf, width); +} + +void RAWToRGB24Row_LASX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { + static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, + 9, 0, 0, 0, 0, 6, 5, 4, 9, 8, 7, + 12, 11, 10, 15, 14, 13, 0, 0, 0, 0}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "vld $vr5, %[shuf], 16\n\t" + "xvpermi.q $xr5, $xr5, 0x00\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 12\n\t" + "vld $vr31, %[src], 60\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "vld $vr2, %[src], 24\n\t" + "vld $vr31, %[src], 72\n\t" + "xvpermi.q $xr2, $xr31, 0x02\n\t" + "vld $vr3, %[src], 32\n\t" + "vld $vr31, %[src], 80\n\t" + "xvpermi.q $xr3, $xr31, 0x02\n\t" + "xvshuf.b $xr0, $xr0, $xr0, $xr4\n\t" + "xvshuf.b $xr1, $xr1, $xr1, $xr4\n\t" + "xvshuf.b $xr2, $xr2, $xr2, $xr4\n\t" + "xvshuf.b $xr3, $xr3, $xr3, $xr5\n\t" + "vstelm.d $vr0, %[dst], 0, 0\n\t" + "xvpermi.q $xr31, $xr0, 0x01\n\t" + "vstelm.d $vr31, %[dst], 48, 0\n\t" + "vstelm.w $vr0, %[dst], 8, 2\n\t" + "xvpermi.q $xr31, $xr0, 0x01\n\t" + "vstelm.w $vr31, %[dst], 56, 2\n\t" + "vstelm.w $vr1, %[dst], 12, 0\n\t" + "vstelm.w $vr1, %[dst], 16, 1\n\t" + "xvpermi.q $xr31, $xr1, 0x01\n\t" + "vstelm.w $vr31, %[dst], 60, 0\n\t" + "vstelm.w $vr31, %[dst], 64, 1\n\t" + "vstelm.w $vr1, %[dst], 20, 2\n\t" + "xvpermi.q $xr31, $xr1, 0x01\n\t" + "vstelm.w $vr31, %[dst], 68, 2\n\t" + "vstelm.d $vr2, %[dst], 24, 0\n\t" + "xvpermi.q $xr31, $xr2, 0x01\n\t" + "vstelm.d $vr31, %[dst], 72, 0\n\t" + "vstelm.w $vr2, %[dst], 32, 2\n\t" + "xvpermi.q $xr31, $xr2, 0x01\n\t" + "vstelm.w $vr31, %[dst], 80, 2\n\t" + "vstelm.w $vr3, %[dst], 36, 0\n\t" + "vstelm.w $vr3, %[dst], 40, 1\n\t" + "xvpermi.q $xr31, $xr3, 0x01\n\t" + "vstelm.w $vr31, %[dst], 84, 0\n\t" + "vstelm.w $vr31, %[dst], 88, 1\n\t" + "vstelm.w $vr3, %[dst], 44, 2\n\t" + "xvpermi.q $xr31, $xr3, 0x01\n\t" + "vstelm.w $vr31, %[dst], 92, 2\n\t" + "addi.d %[src], %[src], 96\n\t" + "addi.d %[dst], %[dst], 96\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_raw), [dst] "+r"(dst_rgb24), [cnt] "+r"(width) + : [shuf] "r"(kShuf) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr31"); +} + +void ARGBToRGB565Row_LASX(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + asm volatile( + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "xvldi $xr7, 63\n\t" + "xvsrli.h $xr7, $xr7, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvsrli.b $xr2, $xr2, 3\n\t" + "xvsrli.b $xr3, $xr3, 2\n\t" + "xvand.v $xr4, $xr2, $xr6\n\t" + "xvsrli.h $xr2, $xr2, 8\n\t" + "xvand.v $xr3, $xr3, $xr7\n\t" + "xvslli.h $xr3, $xr3, 5\n\t" + "xvslli.h $xr2, $xr2, 11\n\t" + "xvor.v $xr4, $xr4, $xr3\n\t" + "xvor.v $xr4, $xr4, $xr2\n\t" + "xvst $xr4, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr7", + "$xr31"); +} + +void ARGBToRGB565DitherRow_LASX(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width) { + asm volatile( + "xvreplgr2vr.w $xr5, %[dith]\n\t" + "xvilvl.b $xr5, $xr5, $xr5\n\t" + "xvilvl.h $xr5, $xr5, $xr5\n\t" + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "xvldi $xr7, 63\n\t" + "xvsrli.h $xr7, $xr7, 8\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvsadd.bu $xr0, $xr0, $xr5\n\t" + "xvpickev.b $xr2, $xr0, $xr0\n\t" + "xvpickod.b $xr3, $xr0, $xr0\n\t" + "xvsrli.b $xr2, $xr2, 3\n\t" + "xvsrli.b $xr3, $xr3, 2\n\t" + "xvand.v $xr4, $xr2, $xr6\n\t" + "xvsrli.h $xr2, $xr2, 8\n\t" + "xvand.v $xr3, $xr3, $xr7\n\t" + "xvslli.h $xr3, $xr3, 5\n\t" + "xvslli.h $xr2, $xr2, 11\n\t" + "xvor.v $xr4, $xr4, $xr3\n\t" + "xvor.v $xr4, $xr4, $xr2\n\t" + "xvpermi.d $xr31, $xr4, 0xd8\n\t" + "vst $vr31, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : [dith] "r"(dither4) + : "memory", "$xr0", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", + "$xr31"); +} + +void ARGBToARGB1555Row_LASX(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + asm volatile( + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvsrli.b $xr2, $xr2, 3\n\t" + "xvand.v $xr4, $xr2, $xr6\n\t" + "xvsrli.h $xr2, $xr2, 8\n\t" + "xvsrli.h $xr5, $xr3, 15\n\t" + "xvsrli.b $xr3, $xr3, 3\n\t" + "xvand.v $xr3, $xr3, $xr6\n\t" + "xvslli.h $xr3, $xr3, 5\n\t" + "xvslli.h $xr2, $xr2, 10\n\t" + "xvslli.h $xr5, $xr5, 15\n\t" + "xvor.v $xr4, $xr4, $xr3\n\t" + "xvor.v $xr4, $xr4, $xr2\n\t" + "xvor.v $xr4, $xr4, $xr5\n\t" + "xvst $xr4, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr31"); +} + +void ARGBToARGB4444Row_LASX(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvsrli.b $xr2, $xr2, 4\n\t" + "xvsrli.b $xr3, $xr3, 4\n\t" + "xvslli.b $xr3, $xr3, 4\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr31"); +} + +void RGB565ToARGBRow_LASX(const uint8_t* src_rgb565, + uint8_t* dst_argb, + int width) { + asm volatile( + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "xvldi $xr7, 63\n\t" + "xvsrli.h $xr7, $xr7, 8\n\t" + "xvldi $xr5, 0xff\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr7\n\t" + "xvsrli.h $xr3, $xr0, 11\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 2\n\t" + "xvsrli.h $xr2, $xr2, 4\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvpickev.b $xr1, $xr1, $xr1\n\t" + "xvpickev.b $xr2, $xr2, $xr2\n\t" + "xvpickev.b $xr3, $xr3, $xr3\n\t" + "xvilvl.b $xr8, $xr2, $xr1\n\t" + "xvilvl.b $xr9, $xr5, $xr3\n\t" + "xvilvl.h $xr10, $xr9, $xr8\n\t" + "xvilvh.h $xr11, $xr9, $xr8\n\t" + "vst $vr10, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr10, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr11, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr11, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgb565), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr31"); +} + +void ARGB1555ToARGBRow_LASX(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width) { + asm volatile( + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvsrli.h $xr3, $xr0, 10\n\t" + "xvand.v $xr3, $xr3, $xr6\n\t" + "xvsrai.h $xr5, $xr0, 15\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 3\n\t" + "xvsrli.h $xr2, $xr2, 2\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvpickev.b $xr1, $xr1, $xr1\n\t" + "xvpickev.b $xr2, $xr2, $xr2\n\t" + "xvpickev.b $xr3, $xr3, $xr3\n\t" + "xvpickev.b $xr5, $xr5, $xr5\n\t" + "xvilvl.b $xr8, $xr2, $xr1\n\t" + "xvilvl.b $xr9, $xr5, $xr3\n\t" + "xvilvl.h $xr10, $xr9, $xr8\n\t" + "xvilvh.h $xr11, $xr9, $xr8\n\t" + "vst $vr10, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr10, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr11, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr11, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb1555), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr8", "$xr9", "$xr10", "$xr11", "$xr31"); +} + +void ARGB4444ToARGBRow_LASX(const uint8_t* src_argb4444, + uint8_t* dst_argb, + int width) { + asm volatile( + "xvldi $xr6, 0x0f\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.b $xr2, $xr0, 4\n\t" + "xvslli.b $xr3, $xr1, 4\n\t" + "xvor.v $xr1, $xr1, $xr3\n\t" + "xvslli.b $xr3, $xr2, 4\n\t" + "xvor.v $xr2, $xr2, $xr3\n\t" + "xvilvl.b $xr4, $xr2, $xr1\n\t" + "xvilvh.b $xr5, $xr2, $xr1\n\t" + "vst $vr4, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr4, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr5, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr5, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb4444), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr31"); +} + +void ARGBMultiplyRow_LASX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvmulwev.h.bu $xr2, $xr0, $xr1\n\t" + "xvmulwod.h.bu $xr3, $xr0, $xr1\n\t" + "xvsrari.h $xr2, $xr2, 8\n\t" + "xvsrari.h $xr3, $xr3, 8\n\t" + "xvpackev.b $xr2, $xr3, $xr2\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3"); +} + +void ARGBAddRow_LASX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvsadd.bu $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} + +void ARGBSubtractRow_LASX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvssub.bu $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} + +void ARGBAttenuateRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint8_t kAlpha[16] = {3, 3, 3, 16, 7, 7, 7, 16, + 11, 11, 11, 16, 15, 15, 15, 16}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "xvldi $xr5, 0xff\n\t" + "xvldi $xr6, 255\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvshuf.b $xr1, $xr5, $xr0, $xr4\n\t" + "xvmulwev.h.bu $xr2, $xr0, $xr1\n\t" + "xvmulwod.h.bu $xr3, $xr0, $xr1\n\t" + "xvadd.h $xr2, $xr2, $xr6\n\t" + "xvadd.h $xr3, $xr3, $xr6\n\t" + "xvpackod.b $xr2, $xr3, $xr2\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(kAlpha) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6"); +} + +void ARGBBlendRow_LASX(const uint8_t* src_argb0, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + static const uint8_t kAlpha[16] = {3, 3, 3, 3, 7, 7, 7, 7, + 11, 11, 11, 11, 15, 15, 15, 15}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "xvldi $xr5, 0xff\n\t" + "xvslli.w $xr5, $xr5, 24\n\t" + "xvldi $xr6, 1\n\t" + "xvslli.h $xr6, $xr6, 8\n\t" + "xvldi $xr8, 0\n\t" + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvshuf.b $xr2, $xr0, $xr0, $xr4\n\t" + "xvilvl.b $xr3, $xr8, $xr2\n\t" + "xvilvh.b $xr2, $xr8, $xr2\n\t" + "xvsub.h $xr3, $xr6, $xr3\n\t" + "xvsub.h $xr2, $xr6, $xr2\n\t" + "xvilvl.b $xr7, $xr8, $xr1\n\t" + "xvilvh.b $xr1, $xr8, $xr1\n\t" + "xvmul.h $xr7, $xr7, $xr3\n\t" + "xvmul.h $xr1, $xr1, $xr2\n\t" + "xvsrlni.b.h $xr1, $xr7, 8\n\t" + "xvsadd.bu $xr0, $xr0, $xr1\n\t" + "xvor.v $xr0, $xr0, $xr5\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [shuf] "r"(kAlpha) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8"); +} + +void ARGBShadeRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + uint32_t value) { + asm volatile( + "xvreplgr2vr.w $xr4, %[val]\n\t" + "xvilvl.b $xr5, $xr4, $xr4\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvilvl.b $xr1, $xr0, $xr0\n\t" + "xvilvh.b $xr2, $xr0, $xr0\n\t" + "xvmuh.hu $xr1, $xr1, $xr5\n\t" + "xvmuh.hu $xr2, $xr2, $xr5\n\t" + "xvsrlni.b.h $xr2, $xr1, 8\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(value) + : "memory", "$xr0", "$xr1", "$xr2", "$xr4", "$xr5"); +} + +void ARGBShuffleRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvshuf.b $xr0, $xr0, $xr0, $xr4\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(shuffler) + : "memory", "$xr0", "$xr4"); +} + +void ARGBGrayRow_LASX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + asm volatile( + "xvldi $xr4, 29\n\t" + "xvldi $xr5, 150\n\t" + "xvldi $xr6, 77\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvmulwev.h.bu $xr7, $xr2, $xr4\n\t" + "xvmaddwod.h.bu $xr7, $xr2, $xr6\n\t" + "xvmaddwev.h.bu $xr7, $xr3, $xr5\n\t" + "xvsrlri.h $xr7, $xr7, 8\n\t" + "xvsrli.h $xr3, $xr3, 8\n\t" + "xvslli.h $xr3, $xr3, 8\n\t" + "xvor.v $xr3, $xr7, $xr3\n\t" + "xvslli.h $xr8, $xr7, 8\n\t" + "xvor.v $xr7, $xr7, $xr8\n\t" + "xvilvl.h $xr0, $xr3, $xr7\n\t" + "xvilvh.h $xr1, $xr3, $xr7\n\t" + "vst $vr0, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr0, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr1, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr1, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr31"); +} + +void ARGBSepiaRow_LASX(uint8_t* dst_argb, int width) { + asm volatile( + "xvldi $xr4, 17\n\t" + "xvldi $xr5, 68\n\t" + "xvldi $xr6, 35\n\t" + "xvldi $xr7, 22\n\t" + "xvldi $xr8, 88\n\t" + "xvldi $xr9, 45\n\t" + "xvldi $xr10, 24\n\t" + "xvldi $xr11, 98\n\t" + "xvldi $xr12, 50\n\t" + "1:\n\t" + "vld $vr0, %[dst], 0\n\t" + "vld $vr31, %[dst], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "vld $vr1, %[dst], 16\n\t" + "vld $vr31, %[dst], 48\n\t" + "xvpermi.q $xr1, $xr31, 0x02\n\t" + "xvpickev.b $xr2, $xr1, $xr0\n\t" + "xvpickod.b $xr3, $xr1, $xr0\n\t" + "xvmulwev.h.bu $xr13, $xr2, $xr4\n\t" + "xvmaddwod.h.bu $xr13, $xr2, $xr6\n\t" + "xvmaddwev.h.bu $xr13, $xr3, $xr5\n\t" + "xvmulwev.h.bu $xr14, $xr2, $xr7\n\t" + "xvmaddwod.h.bu $xr14, $xr2, $xr9\n\t" + "xvmaddwev.h.bu $xr14, $xr3, $xr8\n\t" + "xvmulwev.h.bu $xr15, $xr2, $xr10\n\t" + "xvmaddwod.h.bu $xr15, $xr2, $xr12\n\t" + "xvmaddwev.h.bu $xr15, $xr3, $xr11\n\t" + "xvsrli.h $xr13, $xr13, 7\n\t" + "xvsrli.h $xr14, $xr14, 7\n\t" + "xvsrli.h $xr15, $xr15, 7\n\t" + "xvsat.hu $xr13, $xr13, 7\n\t" + "xvsat.hu $xr14, $xr14, 7\n\t" + "xvsat.hu $xr15, $xr15, 7\n\t" + "xvpickev.b $xr13, $xr13, $xr13\n\t" + "xvpickev.b $xr14, $xr14, $xr14\n\t" + "xvpickev.b $xr15, $xr15, $xr15\n\t" + "xvpickod.b $xr3, $xr3, $xr3\n\t" + "xvilvl.b $xr2, $xr14, $xr13\n\t" + "xvilvl.b $xr3, $xr3, $xr15\n\t" + "xvilvl.h $xr0, $xr3, $xr2\n\t" + "xvilvh.h $xr1, $xr3, $xr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr0, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr1, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr1, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15", "$xr31"); +} + +void ARGBColorMatrixRow_LASX(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + asm volatile( + "vldrepl.w $vr4, %[mat], 0\n\t" + "vldrepl.w $vr31, %[mat], 0\n\t" + "xvpermi.q $xr4, $xr31, 0x02\n\t" + "vldrepl.w $vr5, %[mat], 4\n\t" + "vldrepl.w $vr31, %[mat], 4\n\t" + "xvpermi.q $xr5, $xr31, 0x02\n\t" + "vldrepl.w $vr6, %[mat], 8\n\t" + "vldrepl.w $vr31, %[mat], 8\n\t" + "xvpermi.q $xr6, $xr31, 0x02\n\t" + "vldrepl.w $vr7, %[mat], 12\n\t" + "vldrepl.w $vr31, %[mat], 12\n\t" + "xvpermi.q $xr7, $xr31, 0x02\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvmulwev.h.bu.b $xr8, $xr0, $xr4\n\t" + "xvmaddwod.h.bu.b $xr8, $xr0, $xr4\n\t" + "xvmulwev.h.bu.b $xr9, $xr0, $xr5\n\t" + "xvmaddwod.h.bu.b $xr9, $xr0, $xr5\n\t" + "xvmulwev.h.bu.b $xr10, $xr0, $xr6\n\t" + "xvmaddwod.h.bu.b $xr10, $xr0, $xr6\n\t" + "xvmulwev.h.bu.b $xr11, $xr0, $xr7\n\t" + "xvmaddwod.h.bu.b $xr11, $xr0, $xr7\n\t" + "xvhaddw.w.h $xr8, $xr8, $xr8\n\t" + "xvhaddw.w.h $xr9, $xr9, $xr9\n\t" + "xvhaddw.w.h $xr10, $xr10, $xr10\n\t" + "xvhaddw.w.h $xr11, $xr11, $xr11\n\t" + "xvilvl.w $xr12, $xr9, $xr8\n\t" + "xvilvh.w $xr13, $xr9, $xr8\n\t" + "xvilvl.w $xr14, $xr11, $xr10\n\t" + "xvilvh.w $xr15, $xr11, $xr10\n\t" + "xvilvl.d $xr8, $xr14, $xr12\n\t" + "xvilvh.d $xr9, $xr14, $xr12\n\t" + "xvilvl.d $xr10, $xr15, $xr13\n\t" + "xvilvh.d $xr11, $xr15, $xr13\n\t" + "xvssrani.h.w $xr9, $xr8, 6\n\t" + "xvssrani.h.w $xr11, $xr10, 6\n\t" + "xvssrani.bu.h $xr11, $xr9, 0\n\t" + "xvst $xr11, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [mat] "r"(matrix_argb) + : "memory", "$xr0", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", + "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr31"); +} + +void ARGBQuantizeRow_LASX(uint8_t* dst_argb, + int scale, + int interval_size, + int interval_offset, + int width) { + asm volatile( + "xvreplgr2vr.h $xr4, %[scl]\n\t" + "xvreplgr2vr.b $xr5, %[isz]\n\t" + "xvreplgr2vr.b $xr6, %[ioff]\n\t" + "xvldi $xr7, 0xff\n\t" + "xvslli.w $xr7, $xr7, 24\n\t" + "1:\n\t" + "xvld $xr0, %[dst], 0\n\t" + "xvilvl.b $xr1, $xr0, $xr0\n\t" + "xvilvh.b $xr2, $xr0, $xr0\n\t" + "xvsrli.h $xr1, $xr1, 8\n\t" + "xvsrli.h $xr2, $xr2, 8\n\t" + "xvmuh.hu $xr1, $xr1, $xr4\n\t" + "xvmuh.hu $xr2, $xr2, $xr4\n\t" + "xvsrlni.b.h $xr2, $xr1, 0\n\t" + "xvmul.b $xr2, $xr2, $xr5\n\t" + "xvadd.b $xr2, $xr2, $xr6\n\t" + "xvand.v $xr0, $xr0, $xr7\n\t" + "xvandn.v $xr2, $xr7, $xr2\n\t" + "xvor.v $xr2, $xr2, $xr0\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [scl] "r"(scale), [isz] "r"(interval_size), [ioff] "r"(interval_offset) + : "memory", "$xr0", "$xr1", "$xr2", "$xr4", "$xr5", "$xr6", "$xr7"); +} + +void SobelRow_LASX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + asm volatile( + "xvldi $xr4, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sx], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "vld $vr1, %[sy], 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvsadd.bu $xr0, $xr0, $xr1\n\t" + "xvilvl.b $xr2, $xr0, $xr0\n\t" + "xvilvl.b $xr3, $xr4, $xr0\n\t" + "xvilvl.h $xr0, $xr3, $xr2\n\t" + "xvilvh.h $xr1, $xr3, $xr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr0, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr1, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr1, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sx], %[sx], 16\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); +} + +void SobelToPlaneRow_LASX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[sx], 0\n\t" + "xvld $xr1, %[sy], 0\n\t" + "xvsadd.bu $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[sx], %[sx], 32\n\t" + "addi.d %[sy], %[sy], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_y), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); +} + +void SobelXYRow_LASX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + asm volatile( + "xvldi $xr4, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[sx], 0\n\t" + "xvpermi.d $xr0, $xr0, 0xd8\n\t" + "vld $vr1, %[sy], 0\n\t" + "xvpermi.d $xr1, $xr1, 0xd8\n\t" + "xvsadd.bu $xr2, $xr0, $xr1\n\t" + "xvilvl.b $xr1, $xr2, $xr1\n\t" + "xvilvl.b $xr0, $xr4, $xr0\n\t" + "xvilvl.h $xr2, $xr0, $xr1\n\t" + "xvilvh.h $xr3, $xr0, $xr1\n\t" + "vst $vr2, %[dst], 0\n\t" + "xvpermi.q $xr31, $xr2, 0x01\n\t" + "vst $vr31, %[dst], 32\n\t" + "vst $vr3, %[dst], 16\n\t" + "xvpermi.q $xr31, $xr3, 0x01\n\t" + "vst $vr31, %[dst], 48\n\t" + "addi.d %[sx], %[sx], 16\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr31"); +} + +void InterpolateRow_LASX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int32_t source_y_fraction) { + int y1_fraction = source_y_fraction; + const uint8_t* src_ptr1 = src_ptr + src_stride; + if (y1_fraction == 0) { + CopyRow_LASX(src_ptr, dst_ptr, width); + return; + } + if (y1_fraction == 128) { + asm volatile( + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvavgr.bu $xr0, $xr0, $xr1\n\t" + "xvst $xr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), + [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1"); + return; + } + uint16_t y_frac = (uint16_t)((256 - y1_fraction) + (y1_fraction << 8)); + asm volatile( + "xvreplgr2vr.h $xr3, %[yf]\n\t" + "1:\n\t" + "xvld $xr0, %[s0], 0\n\t" + "xvld $xr1, %[s1], 0\n\t" + "xvilvl.b $xr2, $xr1, $xr0\n\t" + "xvilvh.b $xr4, $xr1, $xr0\n\t" + "xvmulwev.h.bu $xr0, $xr2, $xr3\n\t" + "xvmaddwod.h.bu $xr0, $xr2, $xr3\n\t" + "xvmulwev.h.bu $xr1, $xr4, $xr3\n\t" + "xvmaddwod.h.bu $xr1, $xr4, $xr3\n\t" + "xvsrarni.b.h $xr1, $xr0, 8\n\t" + "xvst $xr1, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), + [cnt] "+r"(width) + : [yf] "r"(y_frac) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); +} + +void HalfFloatRow_LASX(const uint16_t* src, + uint16_t* dst, + float scale, + int width) { + float mult = 1.9259299444e-34f * scale; + asm volatile( + "xvldrepl.w $xr3, %[mult], 0\n\t" + "xvldi $xr4, 0\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvilvl.h $xr1, $xr4, $xr0\n\t" + "xvilvh.h $xr2, $xr4, $xr0\n\t" + "xvffint.s.wu $xr1, $xr1\n\t" + "xvffint.s.wu $xr2, $xr2\n\t" + "xvfmul.s $xr1, $xr1, $xr3\n\t" + "xvfmul.s $xr2, $xr2, $xr3\n\t" + "xvsrlni.h.w $xr2, $xr1, 13\n\t" + "xvst $xr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : [mult] "r"(&mult) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4"); } -// ARGB expects first 3 values to contain RGB and 4th value is ignored. void ARGBToYMatrixRow_LASX(const uint8_t* src_argb, uint8_t* dst_y, int width, const struct ArgbConstants* c) { - int32_t shuff[8] = {0, 4, 1, 5, 2, 6, 3, 7}; asm volatile( - "xvldrepl.b $xr0, %3, 0 \n\t" // load rgbconstants - "xvldrepl.b $xr1, %3, 1 \n\t" // load rgbconstants - "xvldrepl.b $xr2, %3, 2 \n\t" // load rgbconstants - "xvldrepl.b $xr21, %3, 3 \n\t" // load rgbconstants - "xvldrepl.h $xr3, %3, 96 \n\t" // load rgbconstants - "xvld $xr20, %4, 0 \n\t" // load shuff - "1: \n\t" - "xvld $xr4, %0, 0 \n\t" - "xvld $xr5, %0, 32 \n\t" - "xvld $xr6, %0, 64 \n\t" - "xvld $xr7, %0, 96 \n\t" // load 32 pixels of - // ARGB - "xvor.v $xr12, $xr3, $xr3 \n\t" - "xvor.v $xr13, $xr3, $xr3 \n\t" - "addi.d %2, %2, -32 \n\t" // 32 processed per - // loop. - "xvpickev.b $xr8, $xr5, $xr4 \n\t" // BR - "xvpickev.b $xr10, $xr7, $xr6 \n\t" - "xvpickod.b $xr9, $xr5, $xr4 \n\t" // GA - "xvpickod.b $xr11, $xr7, $xr6 \n\t" - "xvmaddwev.h.bu $xr12, $xr8, $xr0 \n\t" // B - "xvmaddwev.h.bu $xr13, $xr10, $xr0 \n\t" - "xvmaddwev.h.bu $xr12, $xr9, $xr1 \n\t" // G - "xvmaddwev.h.bu $xr13, $xr11, $xr1 \n\t" - "xvmaddwod.h.bu $xr12, $xr8, $xr2 \n\t" // R - "xvmaddwod.h.bu $xr13, $xr10, $xr2 \n\t" - "xvmaddwod.h.bu $xr12, $xr9, $xr21 \n\t" // A - "xvmaddwod.h.bu $xr13, $xr11, $xr21 \n\t" - "addi.d %0, %0, 128 \n\t" - "xvpickod.b $xr10, $xr13, $xr12 \n\t" - "xvperm.w $xr11, $xr10, $xr20 \n\t" - "xvst $xr11, %1, 0 \n\t" - "addi.d %1, %1, 32 \n\t" - "bnez %2, 1b \n\t" - : "+&r"(src_argb), // %0 - "+&r"(dst_y), // %1 - "+&r"(width) // %2 - : "r"(c), "r"(shuff) + "xvldrepl.b $xr0, %[c], 0\n\t" + "xvldrepl.b $xr1, %[c], 1\n\t" + "xvldrepl.b $xr2, %[c], 2\n\t" + "xvldrepl.b $xr3, %[c], 3\n\t" + "xvldrepl.h $xr4, %[c], 96\n\t" + "1:\n\t" + "vld $vr5, %[src], 0\n\t" + "vld $vr31, %[src], 32\n\t" + "xvpermi.q $xr5, $xr31, 0x02\n\t" + "vld $vr6, %[src], 16\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr6, $xr31, 0x02\n\t" + "xvor.v $xr9, $xr4, $xr4\n\t" + "xvpickev.b $xr7, $xr6, $xr5\n\t" + "xvpickod.b $xr8, $xr6, $xr5\n\t" + "xvmaddwev.h.bu $xr9, $xr7, $xr0\n\t" + "xvmaddwev.h.bu $xr9, $xr8, $xr1\n\t" + "xvmaddwod.h.bu $xr9, $xr7, $xr2\n\t" + "xvmaddwod.h.bu $xr9, $xr8, $xr3\n\t" + "xvpickod.b $xr9, $xr9, $xr9\n\t" + "xvpermi.d $xr31, $xr9, 0xd8\n\t" + "vst $vr31, %[dst], 0\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [c] "r"(c) : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", - "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr20", - "$xr21"); + "$xr7", "$xr8", "$xr9", "$xr31"); } void RGBToYMatrixRow_LASX(const uint8_t* src_rgba, uint8_t* dst_y, int width, const struct ArgbConstants* c) { - static const int8_t shuff[128] = { - 0, 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18, 20, 21, 23, - 0, 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18, 20, 21, 23, - 24, 26, 27, 29, 30, 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, - 24, 26, 27, 29, 30, 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, - 1, 0, 4, 0, 7, 0, 10, 0, 13, 0, 16, 0, 19, 0, 22, 0, - 1, 0, 4, 0, 7, 0, 10, 0, 13, 0, 16, 0, 19, 0, 22, 0, - 25, 0, 28, 0, 31, 0, 2, 0, 5, 0, 8, 0, 11, 0, 14, 0, - 25, 0, 28, 0, 31, 0, 2, 0, 5, 0, 8, 0, 11, 0, 14, 0}; + static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8, + 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8, + 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; asm volatile( - "xvldrepl.b $xr0, %3, 0 \n\t" // load rgbconstants - "xvldrepl.b $xr1, %3, 1 \n\t" // load rgbconstants - "xvldrepl.b $xr2, %3, 2 \n\t" // load rgbconstants - "xvldrepl.h $xr3, %3, 96 \n\t" // load rgbconstants - "xvld $xr4, %4, 0 \n\t" // load shuff - "xvld $xr5, %4, 32 \n\t" - "xvld $xr6, %4, 64 \n\t" - "xvld $xr7, %4, 96 \n\t" - "1: \n\t" - "xvld $xr8, %0, 0 \n\t" - "xvld $xr9, %0, 32 \n\t" - "xvld $xr10, %0, 64 \n\t" // load 32 pixels of - // RGB - "xvor.v $xr12, $xr3, $xr3 \n\t" - "xvor.v $xr13, $xr3, $xr3 \n\t" - "xvor.v $xr11, $xr9, $xr9 \n\t" - "addi.d %2, %2, -32 \n\t" // 32 processed per - // loop. - "xvpermi.q $xr9, $xr8, 0x30 \n\t" // src0 - "xvpermi.q $xr8, $xr10, 0x03 \n\t" // src1 - "xvpermi.q $xr10, $xr11, 0x30 \n\t" // src2 - "xvshuf.b $xr14, $xr8, $xr9, $xr4 \n\t" - "xvshuf.b $xr15, $xr8, $xr10, $xr5 \n\t" - "xvshuf.b $xr16, $xr8, $xr9, $xr6 \n\t" - "xvshuf.b $xr17, $xr8, $xr10, $xr7 \n\t" - "xvmaddwev.h.bu $xr12, $xr16, $xr1 \n\t" // G - "xvmaddwev.h.bu $xr13, $xr17, $xr1 \n\t" - "xvmaddwev.h.bu $xr12, $xr14, $xr0 \n\t" // B - "xvmaddwev.h.bu $xr13, $xr15, $xr0 \n\t" - "xvmaddwod.h.bu $xr12, $xr14, $xr2 \n\t" // R - "xvmaddwod.h.bu $xr13, $xr15, $xr2 \n\t" - "addi.d %0, %0, 96 \n\t" - "xvpickod.b $xr10, $xr13, $xr12 \n\t" - "xvst $xr10, %1, 0 \n\t" - "addi.d %1, %1, 32 \n\t" - "bnez %2, 1b \n\t" - : "+&r"(src_rgba), // %0 - "+&r"(dst_y), // %1 - "+&r"(width) // %2 - : "r"(c), // %3 - "r"(shuff) // %4 + "xvldrepl.b $xr0, %[c], 0\n\t" + "xvldrepl.b $xr1, %[c], 1\n\t" + "xvldrepl.b $xr2, %[c], 2\n\t" + "xvldrepl.h $xr3, %[c], 96\n\t" + "vld $vr4, %[shuf], 0\n\t" + "xvpermi.q $xr4, $xr4, 0x00\n\t" + "vld $vr5, %[shuf], 16\n\t" + "xvpermi.q $xr5, $xr5, 0x00\n\t" + "1:\n\t" + "vld $vr6, %[src], 0\n\t" + "vld $vr31, %[src], 48\n\t" + "xvpermi.q $xr6, $xr31, 0x02\n\t" + "vld $vr7, %[src], 12\n\t" + "vld $vr31, %[src], 60\n\t" + "xvpermi.q $xr7, $xr31, 0x02\n\t" + "vld $vr8, %[src], 24\n\t" + "vld $vr31, %[src], 72\n\t" + "xvpermi.q $xr8, $xr31, 0x02\n\t" + "vld $vr9, %[src], 32\n\t" + "vld $vr31, %[src], 80\n\t" + "xvpermi.q $xr9, $xr31, 0x02\n\t" + "xvshuf.b $xr6, $xr6, $xr6, $xr4\n\t" + "xvshuf.b $xr7, $xr7, $xr7, $xr4\n\t" + "xvshuf.b $xr8, $xr8, $xr8, $xr4\n\t" + "xvshuf.b $xr9, $xr9, $xr9, $xr5\n\t" + "xvor.v $xr12, $xr3, $xr3\n\t" + "xvor.v $xr13, $xr3, $xr3\n\t" + "xvpickev.b $xr10, $xr7, $xr6\n\t" + "xvpickod.b $xr11, $xr7, $xr6\n\t" + "xvmaddwev.h.bu $xr12, $xr10, $xr0\n\t" + "xvmaddwev.h.bu $xr12, $xr11, $xr1\n\t" + "xvmaddwod.h.bu $xr12, $xr10, $xr2\n\t" + "xvpickev.b $xr10, $xr9, $xr8\n\t" + "xvpickod.b $xr11, $xr9, $xr8\n\t" + "xvmaddwev.h.bu $xr13, $xr10, $xr0\n\t" + "xvmaddwev.h.bu $xr13, $xr11, $xr1\n\t" + "xvmaddwod.h.bu $xr13, $xr10, $xr2\n\t" + "xvpickod.b $xr12, $xr13, $xr12\n\t" + "xvst $xr12, %[dst], 0\n\t" + "addi.d %[src], %[src], 96\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgba), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [c] "r"(c), [shuf] "r"(kShuf) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr31"); +} + +void RGB565ToYRow_LASX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { + asm volatile( + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "xvldi $xr7, 63\n\t" + "xvsrli.h $xr7, $xr7, 8\n\t" + "xvldi $xr8, 25\n\t" + "xvsrli.h $xr8, $xr8, 8\n\t" + "xvldi $xr9, 129\n\t" + "xvsrli.h $xr9, $xr9, 8\n\t" + "xvldi $xr10, 66\n\t" + "xvsrli.h $xr10, $xr10, 8\n\t" + "xvldi $xr11, 33\n\t" + "xvsrli.h $xr11, $xr11, 8\n\t" + "xvslli.h $xr11, $xr11, 7\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr7\n\t" + "xvsrli.h $xr3, $xr0, 11\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 2\n\t" + "xvsrli.h $xr2, $xr2, 4\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvmul.h $xr1, $xr1, $xr8\n\t" + "xvmadd.h $xr1, $xr2, $xr9\n\t" + "xvmadd.h $xr1, $xr3, $xr10\n\t" + "xvadd.h $xr1, $xr1, $xr11\n\t" + "xvpickod.b $xr1, $xr1, $xr1\n\t" + "xvpermi.d $xr31, $xr1, 0xd8\n\t" + "vst $vr31, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgb565), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr6", "$xr7", + "$xr8", "$xr9", "$xr10", "$xr11", "$xr31"); +} + +void ARGB1555ToYRow_LASX(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width) { + asm volatile( + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "xvldi $xr8, 25\n\t" + "xvsrli.h $xr8, $xr8, 8\n\t" + "xvldi $xr9, 129\n\t" + "xvsrli.h $xr9, $xr9, 8\n\t" + "xvldi $xr10, 66\n\t" + "xvsrli.h $xr10, $xr10, 8\n\t" + "xvldi $xr11, 33\n\t" + "xvsrli.h $xr11, $xr11, 8\n\t" + "xvslli.h $xr11, $xr11, 7\n\t" + "1:\n\t" + "xvld $xr0, %[src], 0\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvsrli.h $xr3, $xr0, 10\n\t" + "xvand.v $xr3, $xr3, $xr6\n\t" + "xvsrai.h $xr5, $xr0, 15\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 3\n\t" + "xvsrli.h $xr2, $xr2, 2\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvmul.h $xr1, $xr1, $xr8\n\t" + "xvmadd.h $xr1, $xr2, $xr9\n\t" + "xvmadd.h $xr1, $xr3, $xr10\n\t" + "xvadd.h $xr1, $xr1, $xr11\n\t" + "xvpickod.b $xr1, $xr1, $xr1\n\t" + "xvpermi.d $xr31, $xr1, 0xd8\n\t" + "vst $vr31, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb1555), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr8", "$xr9", "$xr10", "$xr11", "$xr31"); +} + +void RGB565ToUVRow_LASX(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565; + asm volatile( + "xvldi $xr20, 112\n\t" + "xvsrli.h $xr20, $xr20, 8\n\t" + "xvldi $xr21, 74\n\t" + "xvsrli.h $xr21, $xr21, 8\n\t" + "xvldi $xr22, 38\n\t" + "xvsrli.h $xr22, $xr22, 8\n\t" + "xvldi $xr23, 94\n\t" + "xvsrli.h $xr23, $xr23, 8\n\t" + "xvldi $xr24, 18\n\t" + "xvsrli.h $xr24, $xr24, 8\n\t" + "xvldi $xr25, 128\n\t" + "xvsrli.h $xr25, $xr25, 8\n\t" + "xvslli.h $xr25, $xr25, 8\n\t" + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "xvldi $xr7, 63\n\t" + "xvsrli.h $xr7, $xr7, 8\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr31, %[s0], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr7\n\t" + "xvsrli.h $xr3, $xr0, 11\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 2\n\t" + "xvsrli.h $xr2, $xr2, 4\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvor.v $xr10, $xr1, $xr1\n\t" + "xvor.v $xr11, $xr2, $xr2\n\t" + "xvor.v $xr12, $xr3, $xr3\n\t" + "vld $vr0, %[s0], 16\n\t" + "vld $vr31, %[s0], 48\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr7\n\t" + "xvsrli.h $xr3, $xr0, 11\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 2\n\t" + "xvsrli.h $xr2, $xr2, 4\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvpickev.b $xr10, $xr1, $xr10\n\t" + "xvpickev.b $xr11, $xr2, $xr11\n\t" + "xvpickev.b $xr12, $xr3, $xr12\n\t" + "vld $vr0, %[s1], 0\n\t" + "vld $vr31, %[s1], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr7\n\t" + "xvsrli.h $xr3, $xr0, 11\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 2\n\t" + "xvsrli.h $xr2, $xr2, 4\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvor.v $xr13, $xr1, $xr1\n\t" + "xvor.v $xr14, $xr2, $xr2\n\t" + "xvor.v $xr15, $xr3, $xr3\n\t" + "vld $vr0, %[s1], 16\n\t" + "vld $vr31, %[s1], 48\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr7\n\t" + "xvsrli.h $xr3, $xr0, 11\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 2\n\t" + "xvsrli.h $xr2, $xr2, 4\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvpickev.b $xr5, $xr3, $xr15\n\t" + "xvpickev.b $xr4, $xr2, $xr14\n\t" + "xvpickev.b $xr3, $xr1, $xr13\n\t" + "xvor.v $xr0, $xr10, $xr10\n\t" + "xvor.v $xr1, $xr11, $xr11\n\t" + "xvor.v $xr2, $xr12, $xr12\n\t" + "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t" + "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t" + "xvadd.h $xr0, $xr0, $xr3\n\t" + "xvsrari.h $xr0, $xr0, 2\n\t" + "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t" + "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t" + "xvadd.h $xr1, $xr1, $xr4\n\t" + "xvsrari.h $xr1, $xr1, 2\n\t" + "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t" + "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t" + "xvadd.h $xr2, $xr2, $xr5\n\t" + "xvsrari.h $xr2, $xr2, 2\n\t" + "xvor.v $xr6, $xr25, $xr25\n\t" + "xvor.v $xr7, $xr25, $xr25\n\t" + "xvmadd.h $xr6, $xr0, $xr20\n\t" + "xvmsub.h $xr6, $xr1, $xr21\n\t" + "xvmsub.h $xr6, $xr2, $xr22\n\t" + "xvmadd.h $xr7, $xr2, $xr20\n\t" + "xvmsub.h $xr7, $xr1, $xr23\n\t" + "xvmsub.h $xr7, $xr0, $xr24\n\t" + "xvpickod.b $xr6, $xr7, $xr6\n\t" + "xvpermi.d $xr31, $xr6, 0xd8\n\t" + "vst $vr31, %[du], 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "xvpermi.q $xr31, $xr6, 0x01\n\t" + "vstelm.d $vr31, %[dv], 8, 1\n\t" + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "xvldi $xr7, 63\n\t" + "xvsrli.h $xr7, $xr7, 8\n\t" + "addi.d %[s0], %[s0], 64\n\t" + "addi.d %[s1], %[s1], 64\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_rgb565), [s1] "+r"(next_rgb565), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr20", + "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31"); +} + +void ARGB1555ToUVRow_LASX(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555; + asm volatile( + "xvldi $xr20, 112\n\t" + "xvsrli.h $xr20, $xr20, 8\n\t" + "xvldi $xr21, 74\n\t" + "xvsrli.h $xr21, $xr21, 8\n\t" + "xvldi $xr22, 38\n\t" + "xvsrli.h $xr22, $xr22, 8\n\t" + "xvldi $xr23, 94\n\t" + "xvsrli.h $xr23, $xr23, 8\n\t" + "xvldi $xr24, 18\n\t" + "xvsrli.h $xr24, $xr24, 8\n\t" + "xvldi $xr25, 128\n\t" + "xvsrli.h $xr25, $xr25, 8\n\t" + "xvslli.h $xr25, $xr25, 8\n\t" + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr31, %[s0], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvsrli.h $xr3, $xr0, 10\n\t" + "xvand.v $xr3, $xr3, $xr6\n\t" + "xvsrai.h $xr5, $xr0, 15\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 3\n\t" + "xvsrli.h $xr2, $xr2, 2\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvor.v $xr10, $xr1, $xr1\n\t" + "xvor.v $xr11, $xr2, $xr2\n\t" + "xvor.v $xr12, $xr3, $xr3\n\t" + "vld $vr0, %[s0], 16\n\t" + "vld $vr31, %[s0], 48\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvsrli.h $xr3, $xr0, 10\n\t" + "xvand.v $xr3, $xr3, $xr6\n\t" + "xvsrai.h $xr5, $xr0, 15\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 3\n\t" + "xvsrli.h $xr2, $xr2, 2\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvpickev.b $xr10, $xr1, $xr10\n\t" + "xvpickev.b $xr11, $xr2, $xr11\n\t" + "xvpickev.b $xr12, $xr3, $xr12\n\t" + "vld $vr0, %[s1], 0\n\t" + "vld $vr31, %[s1], 32\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvsrli.h $xr3, $xr0, 10\n\t" + "xvand.v $xr3, $xr3, $xr6\n\t" + "xvsrai.h $xr5, $xr0, 15\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 3\n\t" + "xvsrli.h $xr2, $xr2, 2\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvor.v $xr13, $xr1, $xr1\n\t" + "xvor.v $xr14, $xr2, $xr2\n\t" + "xvor.v $xr15, $xr3, $xr3\n\t" + "vld $vr0, %[s1], 16\n\t" + "vld $vr31, %[s1], 48\n\t" + "xvpermi.q $xr0, $xr31, 0x02\n\t" + "xvand.v $xr1, $xr0, $xr6\n\t" + "xvsrli.h $xr2, $xr0, 5\n\t" + "xvand.v $xr2, $xr2, $xr6\n\t" + "xvsrli.h $xr3, $xr0, 10\n\t" + "xvand.v $xr3, $xr3, $xr6\n\t" + "xvsrai.h $xr5, $xr0, 15\n\t" + "xvslli.h $xr4, $xr1, 3\n\t" + "xvsrli.h $xr1, $xr1, 2\n\t" + "xvor.v $xr1, $xr4, $xr1\n\t" + "xvslli.h $xr4, $xr2, 3\n\t" + "xvsrli.h $xr2, $xr2, 2\n\t" + "xvor.v $xr2, $xr4, $xr2\n\t" + "xvslli.h $xr4, $xr3, 3\n\t" + "xvsrli.h $xr3, $xr3, 2\n\t" + "xvor.v $xr3, $xr4, $xr3\n\t" + "xvpickev.b $xr5, $xr3, $xr15\n\t" + "xvpickev.b $xr4, $xr2, $xr14\n\t" + "xvpickev.b $xr3, $xr1, $xr13\n\t" + "xvor.v $xr0, $xr10, $xr10\n\t" + "xvor.v $xr1, $xr11, $xr11\n\t" + "xvor.v $xr2, $xr12, $xr12\n\t" + "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t" + "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t" + "xvadd.h $xr0, $xr0, $xr3\n\t" + "xvsrari.h $xr0, $xr0, 2\n\t" + "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t" + "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t" + "xvadd.h $xr1, $xr1, $xr4\n\t" + "xvsrari.h $xr1, $xr1, 2\n\t" + "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t" + "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t" + "xvadd.h $xr2, $xr2, $xr5\n\t" + "xvsrari.h $xr2, $xr2, 2\n\t" + "xvor.v $xr6, $xr25, $xr25\n\t" + "xvor.v $xr7, $xr25, $xr25\n\t" + "xvmadd.h $xr6, $xr0, $xr20\n\t" + "xvmsub.h $xr6, $xr1, $xr21\n\t" + "xvmsub.h $xr6, $xr2, $xr22\n\t" + "xvmadd.h $xr7, $xr2, $xr20\n\t" + "xvmsub.h $xr7, $xr1, $xr23\n\t" + "xvmsub.h $xr7, $xr0, $xr24\n\t" + "xvpickod.b $xr6, $xr7, $xr6\n\t" + "xvpermi.d $xr31, $xr6, 0xd8\n\t" + "vst $vr31, %[du], 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "xvpermi.q $xr31, $xr6, 0x01\n\t" + "vstelm.d $vr31, %[dv], 8, 1\n\t" + "xvldi $xr6, 31\n\t" + "xvsrli.h $xr6, $xr6, 8\n\t" + "addi.d %[s0], %[s0], 64\n\t" + "addi.d %[s1], %[s1], 64\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb1555), [s1] "+r"(next_argb1555), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", "$xr15", "$xr20", + "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", "$xr31"); +} + +static LIBYUV_NOINLINE void XRGBToUVHelper_LASX(const uint8_t* src, + int stride, + uint8_t* dst_u, + uint8_t* dst_v, + const uint8_t* shuf, + int width) { + const uint8_t* next = src + stride; + asm volatile( + "xvldi $xr20, 112\n\t" + "xvsrli.h $xr20, $xr20, 8\n\t" + "xvldi $xr21, 74\n\t" + "xvsrli.h $xr21, $xr21, 8\n\t" + "xvldi $xr22, 38\n\t" + "xvsrli.h $xr22, $xr22, 8\n\t" + "xvldi $xr23, 94\n\t" + "xvsrli.h $xr23, $xr23, 8\n\t" + "xvldi $xr24, 18\n\t" + "xvsrli.h $xr24, $xr24, 8\n\t" + "xvldi $xr25, 128\n\t" + "xvsrli.h $xr25, $xr25, 8\n\t" + "xvslli.h $xr25, $xr25, 8\n\t" + "vld $vr16, %[shuf], 0\n\t" + "xvpermi.q $xr16, $xr16, 0x00\n\t" + "1:\n\t" + "vld $vr8, %[s0], 0\n\t" + "vld $vr31, %[s0], 64\n\t" + "xvpermi.q $xr8, $xr31, 0x02\n\t" + "vld $vr9, %[s0], 16\n\t" + "vld $vr31, %[s0], 80\n\t" + "xvpermi.q $xr9, $xr31, 0x02\n\t" + "vld $vr10, %[s0], 32\n\t" + "vld $vr31, %[s0], 96\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "vld $vr11, %[s0], 48\n\t" + "vld $vr31, %[s0], 112\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t" + "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t" + "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t" + "xvshuf.b $xr11, $xr11, $xr11, $xr16\n\t" + "xvpickev.b $xr12, $xr9, $xr8\n\t" + "xvpickod.b $xr13, $xr9, $xr8\n\t" + "xvpickev.b $xr14, $xr11, $xr10\n\t" + "xvpickod.b $xr15, $xr11, $xr10\n\t" + "xvpickev.b $xr0, $xr14, $xr12\n\t" + "xvpickod.b $xr2, $xr14, $xr12\n\t" + "xvpickev.b $xr1, $xr15, $xr13\n\t" + "vld $vr8, %[s1], 0\n\t" + "vld $vr31, %[s1], 64\n\t" + "xvpermi.q $xr8, $xr31, 0x02\n\t" + "vld $vr9, %[s1], 16\n\t" + "vld $vr31, %[s1], 80\n\t" + "xvpermi.q $xr9, $xr31, 0x02\n\t" + "vld $vr10, %[s1], 32\n\t" + "vld $vr31, %[s1], 96\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "vld $vr11, %[s1], 48\n\t" + "vld $vr31, %[s1], 112\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t" + "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t" + "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t" + "xvshuf.b $xr11, $xr11, $xr11, $xr16\n\t" + "xvpickev.b $xr12, $xr9, $xr8\n\t" + "xvpickod.b $xr13, $xr9, $xr8\n\t" + "xvpickev.b $xr14, $xr11, $xr10\n\t" + "xvpickod.b $xr15, $xr11, $xr10\n\t" + "xvpickev.b $xr3, $xr14, $xr12\n\t" + "xvpickod.b $xr5, $xr14, $xr12\n\t" + "xvpickev.b $xr4, $xr15, $xr13\n\t" + "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t" + "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t" + "xvadd.h $xr0, $xr0, $xr3\n\t" + "xvsrari.h $xr0, $xr0, 2\n\t" + "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t" + "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t" + "xvadd.h $xr1, $xr1, $xr4\n\t" + "xvsrari.h $xr1, $xr1, 2\n\t" + "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t" + "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t" + "xvadd.h $xr2, $xr2, $xr5\n\t" + "xvsrari.h $xr2, $xr2, 2\n\t" + "xvor.v $xr6, $xr25, $xr25\n\t" + "xvor.v $xr7, $xr25, $xr25\n\t" + "xvmadd.h $xr6, $xr0, $xr20\n\t" + "xvmsub.h $xr6, $xr1, $xr21\n\t" + "xvmsub.h $xr6, $xr2, $xr22\n\t" + "xvmadd.h $xr7, $xr2, $xr20\n\t" + "xvmsub.h $xr7, $xr1, $xr23\n\t" + "xvmsub.h $xr7, $xr0, $xr24\n\t" + "xvpickod.b $xr6, $xr7, $xr6\n\t" + "xvpermi.d $xr31, $xr6, 0xd8\n\t" + "vst $vr31, %[du], 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "xvpermi.q $xr31, $xr6, 0x01\n\t" + "vstelm.d $vr31, %[dv], 8, 1\n\t" + "addi.d %[s0], %[s0], 128\n\t" + "addi.d %[s1], %[s1], 128\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : [shuf] "r"(shuf) : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", - "$xr15", "$xr16", "$xr17"); + "$xr15", "$xr16", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", "$xr25", + "$xr31"); +} + +void BGRAToUVRow_LASX(const uint8_t* src_bgra, + int src_stride_bgra, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[16] = {3, 2, 1, 0, 7, 6, 5, 4, + 11, 10, 9, 8, 15, 14, 13, 12}; + XRGBToUVHelper_LASX(src_bgra, src_stride_bgra, dst_u, dst_v, kShuf, width); +} + +void ABGRToUVRow_LASX(const uint8_t* src_abgr, + int src_stride_abgr, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[16] = {2, 1, 0, 3, 6, 5, 4, 7, + 10, 9, 8, 11, 14, 13, 12, 15}; + XRGBToUVHelper_LASX(src_abgr, src_stride_abgr, dst_u, dst_v, kShuf, width); +} + +void RGBAToUVRow_LASX(const uint8_t* src_rgba, + int src_stride_rgba, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[16] = {1, 2, 3, 0, 5, 6, 7, 4, + 9, 10, 11, 8, 13, 14, 15, 12}; + XRGBToUVHelper_LASX(src_rgba, src_stride_rgba, dst_u, dst_v, kShuf, width); +} + +static LIBYUV_NOINLINE void RGB24ToUVHelper_LASX(const uint8_t* src, + int stride, + uint8_t* dst_u, + uint8_t* dst_v, + const uint8_t* shuf, + int width) { + const uint8_t* next = src + stride; + asm volatile( + "xvldi $xr20, 112\n\t" + "xvsrli.h $xr20, $xr20, 8\n\t" + "xvldi $xr21, 74\n\t" + "xvsrli.h $xr21, $xr21, 8\n\t" + "xvldi $xr22, 38\n\t" + "xvsrli.h $xr22, $xr22, 8\n\t" + "xvldi $xr23, 94\n\t" + "xvsrli.h $xr23, $xr23, 8\n\t" + "xvldi $xr24, 18\n\t" + "xvsrli.h $xr24, $xr24, 8\n\t" + "xvldi $xr25, 128\n\t" + "xvsrli.h $xr25, $xr25, 8\n\t" + "xvslli.h $xr25, $xr25, 8\n\t" + "vld $vr16, %[shuf], 0\n\t" + "xvpermi.q $xr16, $xr16, 0x00\n\t" + "vld $vr17, %[shuf], 16\n\t" + "xvpermi.q $xr17, $xr17, 0x00\n\t" + "1:\n\t" + "vld $vr8, %[s0], 0\n\t" + "vld $vr31, %[s0], 48\n\t" + "xvpermi.q $xr8, $xr31, 0x02\n\t" + "vld $vr9, %[s0], 12\n\t" + "vld $vr31, %[s0], 60\n\t" + "xvpermi.q $xr9, $xr31, 0x02\n\t" + "vld $vr10, %[s0], 24\n\t" + "vld $vr31, %[s0], 72\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "vld $vr11, %[s0], 32\n\t" + "vld $vr31, %[s0], 80\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t" + "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t" + "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t" + "xvshuf.b $xr11, $xr11, $xr11, $xr17\n\t" + "xvpickev.b $xr12, $xr9, $xr8\n\t" + "xvpickod.b $xr13, $xr9, $xr8\n\t" + "xvpickev.b $xr14, $xr11, $xr10\n\t" + "xvpickod.b $xr15, $xr11, $xr10\n\t" + "xvpickev.b $xr0, $xr14, $xr12\n\t" + "xvpickod.b $xr2, $xr14, $xr12\n\t" + "xvpickev.b $xr1, $xr15, $xr13\n\t" + "vld $vr8, %[s1], 0\n\t" + "vld $vr31, %[s1], 48\n\t" + "xvpermi.q $xr8, $xr31, 0x02\n\t" + "vld $vr9, %[s1], 12\n\t" + "vld $vr31, %[s1], 60\n\t" + "xvpermi.q $xr9, $xr31, 0x02\n\t" + "vld $vr10, %[s1], 24\n\t" + "vld $vr31, %[s1], 72\n\t" + "xvpermi.q $xr10, $xr31, 0x02\n\t" + "vld $vr11, %[s1], 32\n\t" + "vld $vr31, %[s1], 80\n\t" + "xvpermi.q $xr11, $xr31, 0x02\n\t" + "xvshuf.b $xr8, $xr8, $xr8, $xr16\n\t" + "xvshuf.b $xr9, $xr9, $xr9, $xr16\n\t" + "xvshuf.b $xr10, $xr10, $xr10, $xr16\n\t" + "xvshuf.b $xr11, $xr11, $xr11, $xr17\n\t" + "xvpickev.b $xr12, $xr9, $xr8\n\t" + "xvpickod.b $xr13, $xr9, $xr8\n\t" + "xvpickev.b $xr14, $xr11, $xr10\n\t" + "xvpickod.b $xr15, $xr11, $xr10\n\t" + "xvpickev.b $xr3, $xr14, $xr12\n\t" + "xvpickod.b $xr5, $xr14, $xr12\n\t" + "xvpickev.b $xr4, $xr15, $xr13\n\t" + "xvhaddw.hu.bu $xr0, $xr0, $xr0\n\t" + "xvhaddw.hu.bu $xr3, $xr3, $xr3\n\t" + "xvadd.h $xr0, $xr0, $xr3\n\t" + "xvsrari.h $xr0, $xr0, 2\n\t" + "xvhaddw.hu.bu $xr1, $xr1, $xr1\n\t" + "xvhaddw.hu.bu $xr4, $xr4, $xr4\n\t" + "xvadd.h $xr1, $xr1, $xr4\n\t" + "xvsrari.h $xr1, $xr1, 2\n\t" + "xvhaddw.hu.bu $xr2, $xr2, $xr2\n\t" + "xvhaddw.hu.bu $xr5, $xr5, $xr5\n\t" + "xvadd.h $xr2, $xr2, $xr5\n\t" + "xvsrari.h $xr2, $xr2, 2\n\t" + "xvor.v $xr6, $xr25, $xr25\n\t" + "xvor.v $xr7, $xr25, $xr25\n\t" + "xvmadd.h $xr6, $xr0, $xr20\n\t" + "xvmsub.h $xr6, $xr1, $xr21\n\t" + "xvmsub.h $xr6, $xr2, $xr22\n\t" + "xvmadd.h $xr7, $xr2, $xr20\n\t" + "xvmsub.h $xr7, $xr1, $xr23\n\t" + "xvmsub.h $xr7, $xr0, $xr24\n\t" + "xvpickod.b $xr6, $xr7, $xr6\n\t" + "xvpermi.d $xr31, $xr6, 0xd8\n\t" + "vst $vr31, %[du], 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "xvpermi.q $xr31, $xr6, 0x01\n\t" + "vstelm.d $vr31, %[dv], 8, 1\n\t" + "addi.d %[s0], %[s0], 96\n\t" + "addi.d %[s1], %[s1], 96\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -32\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$xr0", "$xr1", "$xr2", "$xr3", "$xr4", "$xr5", "$xr6", + "$xr7", "$xr8", "$xr9", "$xr10", "$xr11", "$xr12", "$xr13", "$xr14", + "$xr15", "$xr16", "$xr17", "$xr20", "$xr21", "$xr22", "$xr23", "$xr24", + "$xr25", "$xr31"); +} + +void RGB24ToUVRow_LASX(const uint8_t* src_rgb24, + int src_stride_rgb24, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8, + 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8, + 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; + RGB24ToUVHelper_LASX(src_rgb24, src_stride_rgb24, dst_u, dst_v, kShuf, width); +} + +void RAWToUVRow_LASX(const uint8_t* src_raw, + int src_stride_raw, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[32] = {2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, + 0, 11, 10, 9, 0, 6, 5, 4, 0, 9, 8, + 7, 0, 12, 11, 10, 0, 15, 14, 13, 0}; + RGB24ToUVHelper_LASX(src_raw, src_stride_raw, dst_u, dst_v, kShuf, width); } -// undef for unified sources build -#undef ALPHA_VAL -#undef YUVTORGB_SETUP -#undef READYUV422_D -#undef READYUV422 -#undef YUVTORGB_D -#undef YUVTORGB -#undef STOREARGB_D -#undef STOREARGB -#undef RGBTOUV #ifdef __cplusplus } // extern "C" diff --git a/source/row_lsx.cc b/source/row_lsx.cc index b4fdaaf69..539567990 100644 --- a/source/row_lsx.cc +++ b/source/row_lsx.cc @@ -1,8 +1,6 @@ /* * Copyright 2022 The LibYuv Project Authors. All rights reserved. * - * Copyright (c) 2022 Loongson Technology Corporation Limited - * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found @@ -12,335 +10,201 @@ #include "libyuv/row.h" -// Note: every register written by an inline asm block (vector, general -// purpose, mask/predicate, "cc" for flags and "memory" for stores) must be -// listed in its clobber list, even when the ABI treats it as caller-saved. -// With LTO/LTCG the compiler may keep values live in any register across the -// asm statement, and other OS ABIs (e.g. Windows) differ in which registers -// are callee-saved. List "memory" first, then "cc", then registers. - #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) -#include "libyuv/loongson_intrinsics.h" + +#include "libyuv/basic_types.h" #ifdef __cplusplus namespace libyuv { extern "C" { #endif -// Fill YUV -> RGB conversion constants into vectors -#define YUVTORGB_SETUP(yuvconst, vr, ub, vg, ug, yg, yb) \ - { \ - ub = __lsx_vreplgr2vr_h(yuvconst->kUVToB[0]); \ - vr = __lsx_vreplgr2vr_h(yuvconst->kUVToR[1]); \ - ug = __lsx_vreplgr2vr_h(yuvconst->kUVToG[0]); \ - vg = __lsx_vreplgr2vr_h(yuvconst->kUVToG[1]); \ - yg = __lsx_vreplgr2vr_h(yuvconst->kYToRgb[0]); \ - yb = __lsx_vreplgr2vr_w(yuvconst->kYBiasToRgb[0]); \ - } +void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int count) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(count) + : + : "memory", "$vr0"); +} -// Load 32 YUV422 pixel data -#define READYUV422_D(psrc_y, psrc_u, psrc_v, out_y, uv_l, uv_h) \ - { \ - __m128i temp0, temp1; \ - \ - DUP2_ARG2(__lsx_vld, psrc_y, 0, psrc_u, 0, out_y, temp0); \ - temp1 = __lsx_vld(psrc_v, 0); \ - temp0 = __lsx_vsub_b(temp0, const_80); \ - temp1 = __lsx_vsub_b(temp1, const_80); \ - temp0 = __lsx_vsllwil_h_b(temp0, 0); \ - temp1 = __lsx_vsllwil_h_b(temp1, 0); \ - uv_l = __lsx_vilvl_h(temp0, temp1); \ - uv_h = __lsx_vilvh_h(temp0, temp1); \ - } +void SetRow_LSX(uint8_t* dst, uint8_t v8, int count) { + asm volatile( + "vreplgr2vr.b $vr0, %[val]\n\t" + "1:\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst), [cnt] "+r"(count) + : [val] "r"(v8) + : "memory", "$vr0"); +} -// Load 16 YUV422 pixel data -#define READYUV422(psrc_y, psrc_u, psrc_v, out_y, uv) \ - { \ - __m128i temp0, temp1; \ - \ - out_y = __lsx_vld(psrc_y, 0); \ - temp0 = __lsx_vldrepl_d(psrc_u, 0); \ - temp1 = __lsx_vldrepl_d(psrc_v, 0); \ - uv = __lsx_vilvl_b(temp0, temp1); \ - uv = __lsx_vsub_b(uv, const_80); \ - uv = __lsx_vsllwil_h_b(uv, 0); \ - } +void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width) { + asm volatile( + "vreplgr2vr.w $vr0, %[val]\n\t" + "1:\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(v32) + : "memory", "$vr0"); +} -// Convert 16 pixels of YUV420 to RGB. -#define YUVTORGB_D(in_y, in_uvl, in_uvh, ubvr, ugvg, yg, yb, b_l, b_h, g_l, \ - g_h, r_l, r_h) \ - { \ - __m128i u_l, u_h, v_l, v_h; \ - __m128i yl_ev, yl_od, yh_ev, yh_od; \ - __m128i temp0, temp1, temp2, temp3; \ - \ - temp0 = __lsx_vilvl_b(in_y, in_y); \ - temp1 = __lsx_vilvh_b(in_y, in_y); \ - yl_ev = __lsx_vmulwev_w_hu_h(temp0, yg); \ - yl_od = __lsx_vmulwod_w_hu_h(temp0, yg); \ - yh_ev = __lsx_vmulwev_w_hu_h(temp1, yg); \ - yh_od = __lsx_vmulwod_w_hu_h(temp1, yg); \ - DUP4_ARG2(__lsx_vsrai_w, yl_ev, 16, yl_od, 16, yh_ev, 16, yh_od, 16, \ - yl_ev, yl_od, yh_ev, yh_od); \ - yl_ev = __lsx_vadd_w(yl_ev, yb); \ - yl_od = __lsx_vadd_w(yl_od, yb); \ - yh_ev = __lsx_vadd_w(yh_ev, yb); \ - yh_od = __lsx_vadd_w(yh_od, yb); \ - v_l = __lsx_vmulwev_w_h(in_uvl, ubvr); \ - u_l = __lsx_vmulwod_w_h(in_uvl, ubvr); \ - v_h = __lsx_vmulwev_w_h(in_uvh, ubvr); \ - u_h = __lsx_vmulwod_w_h(in_uvh, ubvr); \ - temp0 = __lsx_vadd_w(yl_ev, u_l); \ - temp1 = __lsx_vadd_w(yl_od, u_l); \ - temp2 = __lsx_vadd_w(yh_ev, u_h); \ - temp3 = __lsx_vadd_w(yh_od, u_h); \ - DUP4_ARG2(__lsx_vsrai_w, temp0, 6, temp1, 6, temp2, 6, temp3, 6, temp0, \ - temp1, temp2, temp3); \ - DUP4_ARG1(__lsx_vclip255_w, temp0, temp1, temp2, temp3, temp0, temp1, \ - temp2, temp3); \ - b_l = __lsx_vpackev_h(temp1, temp0); \ - b_h = __lsx_vpackev_h(temp3, temp2); \ - temp0 = __lsx_vadd_w(yl_ev, v_l); \ - temp1 = __lsx_vadd_w(yl_od, v_l); \ - temp2 = __lsx_vadd_w(yh_ev, v_h); \ - temp3 = __lsx_vadd_w(yh_od, v_h); \ - DUP4_ARG2(__lsx_vsrai_w, temp0, 6, temp1, 6, temp2, 6, temp3, 6, temp0, \ - temp1, temp2, temp3); \ - DUP4_ARG1(__lsx_vclip255_w, temp0, temp1, temp2, temp3, temp0, temp1, \ - temp2, temp3); \ - r_l = __lsx_vpackev_h(temp1, temp0); \ - r_h = __lsx_vpackev_h(temp3, temp2); \ - DUP2_ARG2(__lsx_vdp2_w_h, in_uvl, ugvg, in_uvh, ugvg, u_l, u_h); \ - temp0 = __lsx_vsub_w(yl_ev, u_l); \ - temp1 = __lsx_vsub_w(yl_od, u_l); \ - temp2 = __lsx_vsub_w(yh_ev, u_h); \ - temp3 = __lsx_vsub_w(yh_od, u_h); \ - DUP4_ARG2(__lsx_vsrai_w, temp0, 6, temp1, 6, temp2, 6, temp3, 6, temp0, \ - temp1, temp2, temp3); \ - DUP4_ARG1(__lsx_vclip255_w, temp0, temp1, temp2, temp3, temp0, temp1, \ - temp2, temp3); \ - g_l = __lsx_vpackev_h(temp1, temp0); \ - g_h = __lsx_vpackev_h(temp3, temp2); \ - } - -// Convert 8 pixels of YUV420 to RGB. -#define YUVTORGB(in_y, in_vu, vrub, vgug, yg, yb, out_b, out_g, out_r) \ - { \ - __m128i y_ev, y_od, u_l, v_l; \ - __m128i tmp0, tmp1, tmp2, tmp3; \ - \ - tmp0 = __lsx_vilvl_b(in_y, in_y); \ - y_ev = __lsx_vmulwev_w_hu_h(tmp0, yg); \ - y_od = __lsx_vmulwod_w_hu_h(tmp0, yg); \ - y_ev = __lsx_vsrai_w(y_ev, 16); \ - y_od = __lsx_vsrai_w(y_od, 16); \ - y_ev = __lsx_vadd_w(y_ev, yb); \ - y_od = __lsx_vadd_w(y_od, yb); \ - in_vu = __lsx_vilvl_b(zero, in_vu); \ - in_vu = __lsx_vsub_h(in_vu, const_80); \ - u_l = __lsx_vmulwev_w_h(in_vu, vrub); \ - v_l = __lsx_vmulwod_w_h(in_vu, vrub); \ - tmp0 = __lsx_vadd_w(y_ev, u_l); \ - tmp1 = __lsx_vadd_w(y_od, u_l); \ - tmp2 = __lsx_vadd_w(y_ev, v_l); \ - tmp3 = __lsx_vadd_w(y_od, v_l); \ - tmp0 = __lsx_vsrai_w(tmp0, 6); \ - tmp1 = __lsx_vsrai_w(tmp1, 6); \ - tmp2 = __lsx_vsrai_w(tmp2, 6); \ - tmp3 = __lsx_vsrai_w(tmp3, 6); \ - tmp0 = __lsx_vclip255_w(tmp0); \ - tmp1 = __lsx_vclip255_w(tmp1); \ - tmp2 = __lsx_vclip255_w(tmp2); \ - tmp3 = __lsx_vclip255_w(tmp3); \ - out_b = __lsx_vpackev_h(tmp1, tmp0); \ - out_r = __lsx_vpackev_h(tmp3, tmp2); \ - tmp0 = __lsx_vdp2_w_h(in_vu, vgug); \ - tmp1 = __lsx_vsub_w(y_ev, tmp0); \ - tmp2 = __lsx_vsub_w(y_od, tmp0); \ - tmp1 = __lsx_vsrai_w(tmp1, 6); \ - tmp2 = __lsx_vsrai_w(tmp2, 6); \ - tmp1 = __lsx_vclip255_w(tmp1); \ - tmp2 = __lsx_vclip255_w(tmp2); \ - out_g = __lsx_vpackev_h(tmp2, tmp1); \ - } - -// Convert I444 pixels of YUV420 to RGB. -#define I444TORGB(in_yy, in_u, in_v, ub, vr, ugvg, yg, yb, out_b, out_g, \ - out_r) \ - { \ - __m128i y_ev, y_od, u_ev, v_ev, u_od, v_od; \ - __m128i tmp0, tmp1, tmp2, tmp3; \ - \ - y_ev = __lsx_vmulwev_w_hu_h(in_yy, yg); \ - y_od = __lsx_vmulwod_w_hu_h(in_yy, yg); \ - y_ev = __lsx_vsrai_w(y_ev, 16); \ - y_od = __lsx_vsrai_w(y_od, 16); \ - y_ev = __lsx_vadd_w(y_ev, yb); \ - y_od = __lsx_vadd_w(y_od, yb); \ - in_u = __lsx_vsub_h(in_u, const_80); \ - in_v = __lsx_vsub_h(in_v, const_80); \ - u_ev = __lsx_vmulwev_w_h(in_u, ub); \ - u_od = __lsx_vmulwod_w_h(in_u, ub); \ - v_ev = __lsx_vmulwev_w_h(in_v, vr); \ - v_od = __lsx_vmulwod_w_h(in_v, vr); \ - tmp0 = __lsx_vadd_w(y_ev, u_ev); \ - tmp1 = __lsx_vadd_w(y_od, u_od); \ - tmp2 = __lsx_vadd_w(y_ev, v_ev); \ - tmp3 = __lsx_vadd_w(y_od, v_od); \ - tmp0 = __lsx_vsrai_w(tmp0, 6); \ - tmp1 = __lsx_vsrai_w(tmp1, 6); \ - tmp2 = __lsx_vsrai_w(tmp2, 6); \ - tmp3 = __lsx_vsrai_w(tmp3, 6); \ - tmp0 = __lsx_vclip255_w(tmp0); \ - tmp1 = __lsx_vclip255_w(tmp1); \ - tmp2 = __lsx_vclip255_w(tmp2); \ - tmp3 = __lsx_vclip255_w(tmp3); \ - out_b = __lsx_vpackev_h(tmp1, tmp0); \ - out_r = __lsx_vpackev_h(tmp3, tmp2); \ - u_ev = __lsx_vpackev_h(in_u, in_v); \ - u_od = __lsx_vpackod_h(in_u, in_v); \ - v_ev = __lsx_vdp2_w_h(u_ev, ugvg); \ - v_od = __lsx_vdp2_w_h(u_od, ugvg); \ - tmp0 = __lsx_vsub_w(y_ev, v_ev); \ - tmp1 = __lsx_vsub_w(y_od, v_od); \ - tmp0 = __lsx_vsrai_w(tmp0, 6); \ - tmp1 = __lsx_vsrai_w(tmp1, 6); \ - tmp0 = __lsx_vclip255_w(tmp0); \ - tmp1 = __lsx_vclip255_w(tmp1); \ - out_g = __lsx_vpackev_h(tmp1, tmp0); \ - } - -// Pack and Store 16 ARGB values. -#define STOREARGB_D(a_l, a_h, r_l, r_h, g_l, g_h, b_l, b_h, pdst_argb) \ - { \ - __m128i temp0, temp1, temp2, temp3; \ - temp0 = __lsx_vpackev_b(g_l, b_l); \ - temp1 = __lsx_vpackev_b(a_l, r_l); \ - temp2 = __lsx_vpackev_b(g_h, b_h); \ - temp3 = __lsx_vpackev_b(a_h, r_h); \ - r_l = __lsx_vilvl_h(temp1, temp0); \ - r_h = __lsx_vilvh_h(temp1, temp0); \ - g_l = __lsx_vilvl_h(temp3, temp2); \ - g_h = __lsx_vilvh_h(temp3, temp2); \ - __lsx_vst(r_l, pdst_argb, 0); \ - __lsx_vst(r_h, pdst_argb, 16); \ - __lsx_vst(g_l, pdst_argb, 32); \ - __lsx_vst(g_h, pdst_argb, 48); \ - pdst_argb += 64; \ - } - -// Pack and Store 8 ARGB values. -#define STOREARGB(in_a, in_r, in_g, in_b, pdst_argb) \ - { \ - __m128i temp0, temp1; \ - __m128i dst0, dst1; \ - \ - temp0 = __lsx_vpackev_b(in_g, in_b); \ - temp1 = __lsx_vpackev_b(in_a, in_r); \ - dst0 = __lsx_vilvl_h(temp1, temp0); \ - dst1 = __lsx_vilvh_h(temp1, temp0); \ - __lsx_vst(dst0, pdst_argb, 0); \ - __lsx_vst(dst1, pdst_argb, 16); \ - pdst_argb += 32; \ - } - -#define RGBTOUV(_tmpb, _tmpg, _tmpr, _nexb, _nexg, _nexr, _dst0) \ - { \ - __m128i _tmp0, _tmp1, _tmp2, _tmp3, _tmp4, _tmp5; \ - __m128i _reg0, _reg1; \ - _tmp0 = __lsx_vaddwev_h_bu(_tmpb, _nexb); \ - _tmp1 = __lsx_vaddwod_h_bu(_tmpb, _nexb); \ - _tmp2 = __lsx_vaddwev_h_bu(_tmpg, _nexg); \ - _tmp3 = __lsx_vaddwod_h_bu(_tmpg, _nexg); \ - _reg0 = __lsx_vaddwev_h_bu(_tmpr, _nexr); \ - _reg1 = __lsx_vaddwod_h_bu(_tmpr, _nexr); \ - _tmp4 = __lsx_vaddwev_w_hu(_tmp0, _tmp1); \ - _tmp5 = __lsx_vaddwod_w_hu(_tmp0, _tmp1); \ - _tmp0 = __lsx_vilvl_w(_tmp5, _tmp4); \ - _tmp1 = __lsx_vilvh_w(_tmp5, _tmp4); \ - _tmpb = __lsx_vssrarni_hu_w(_tmp1, _tmp0, 2); \ - _tmp4 = __lsx_vaddwev_w_hu(_tmp2, _tmp3); \ - _tmp5 = __lsx_vaddwod_w_hu(_tmp2, _tmp3); \ - _tmp2 = __lsx_vilvl_w(_tmp5, _tmp4); \ - _tmp3 = __lsx_vilvh_w(_tmp5, _tmp4); \ - _tmpg = __lsx_vssrarni_hu_w(_tmp3, _tmp2, 2); \ - _tmp4 = __lsx_vaddwev_w_hu(_reg0, _reg1); \ - _tmp5 = __lsx_vaddwod_w_hu(_reg0, _reg1); \ - _tmp0 = __lsx_vilvl_w(_tmp5, _tmp4); \ - _tmp1 = __lsx_vilvh_w(_tmp5, _tmp4); \ - _tmpr = __lsx_vssrarni_hu_w(_tmp1, _tmp0, 2); \ - _reg0 = __lsx_vmadd_h(const_8000, const_112, _tmpb); \ - _reg1 = __lsx_vmadd_h(const_8000, const_112, _tmpr); \ - _reg0 = __lsx_vmsub_h(_reg0, const_74, _tmpg); \ - _reg1 = __lsx_vmsub_h(_reg1, const_94, _tmpg); \ - _reg0 = __lsx_vmsub_h(_reg0, const_38, _tmpr); \ - _reg1 = __lsx_vmsub_h(_reg1, const_18, _tmpb); \ - _dst0 = __lsx_vpickod_b(_reg1, _reg0); \ - } - -void MirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width) { - int x; - int len = width / 32; - __m128i src0, src1; - __m128i shuffler = {0x08090A0B0C0D0E0F, 0x0001020304050607}; - src += width - 32; - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); - DUP2_ARG3(__lsx_vshuf_b, src0, src0, shuffler, src1, src1, shuffler, src0, - src1); - __lsx_vst(src1, dst, 0); - __lsx_vst(src0, dst, 16); - dst += 32; - src -= 32; - } +void MirrorRow_LSX(const uint8_t* src_y, uint8_t* dst_y, int width) { + static const uint8_t kRev[16] = {15, 14, 13, 12, 11, 10, 9, 8, + 7, 6, 5, 4, 3, 2, 1, 0}; + src_y += width - 16; + asm volatile( + "vld $vr1, %[rev], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_y), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [rev] "r"(kRev) + : "memory", "$vr0", "$vr1"); } void MirrorUVRow_LSX(const uint8_t* src_uv, uint8_t* dst_uv, int width) { - int x; - int len = width / 8; - __m128i src, dst; - __m128i shuffler = {0x0004000500060007, 0x0000000100020003}; - - src_uv += (width - 8) << 1; - for (x = 0; x < len; x++) { - src = __lsx_vld(src_uv, 0); - dst = __lsx_vshuf_h(shuffler, src, src); - __lsx_vst(dst, dst_uv, 0); - src_uv -= 16; - dst_uv += 16; - } + static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1}; + src_uv += width * 2 - 16; + asm volatile( + "vld $vr1, %[rev], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uv), [dst] "+r"(dst_uv), [cnt] "+r"(width) + : [rev] "r"(kRevUV) + : "memory", "$vr0", "$vr1"); } -void ARGBMirrorRow_LSX(const uint8_t* src, uint8_t* dst, int width) { - int x; - int len = width / 8; - __m128i src0, src1; - __m128i shuffler = {0x0B0A09080F0E0D0C, 0x0302010007060504}; - - src += (width * 4) - 32; - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); - DUP2_ARG3(__lsx_vshuf_b, src0, src0, shuffler, src1, src1, shuffler, src0, - src1); - __lsx_vst(src1, dst, 0); - __lsx_vst(src0, dst, 16); - dst += 32; - src -= 32; - } +void ARGBMirrorRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + src_argb += width * 4 - 16; + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf4i.w $vr0, $vr0, 0x1b\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], -16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0"); } -void CopyRow_LSX(const uint8_t* src, uint8_t* dst, int width) { - int x; - int len = width / 32; - __m128i src0, src1; +void MirrorSplitUVRow_LSX(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kRevUV[16] = {14, 15, 12, 13, 10, 11, 8, 9, + 6, 7, 4, 5, 2, 3, 0, 1}; + src_uv += width * 2 - 32; + asm volatile( + "vld $vr4, %[rev], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 16\n\t" + "vld $vr1, %[src], 0\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" + "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vst $vr2, %[du], 0\n\t" + "vst $vr3, %[dv], 0\n\t" + "addi.d %[src], %[src], -32\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : + [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width) + : [rev] "r"(kRevUV) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1); - __lsx_vst(src0, dst, 0); - __lsx_vst(src1, dst, 16); - src += 32; - dst += 32; - } +void SplitUVRow_LSX(const uint8_t* src_uv, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vst $vr2, %[du], 0\n\t" + "vst $vr3, %[dv], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[du], %[du], 16\n\t" + "addi.d %[dv], %[dv], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : + [src] "+r"(src_uv), [du] "+r"(dst_u), [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} + +void MergeUVRow_LSX(const uint8_t* src_u, + const uint8_t* src_v, + uint8_t* dst_uv, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[su], 0\n\t" + "vld $vr1, %[sv], 0\n\t" + "vilvl.b $vr2, $vr1, $vr0\n\t" + "vilvh.b $vr3, $vr1, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "vst $vr3, %[dst], 16\n\t" + "addi.d %[su], %[su], 16\n\t" + "addi.d %[sv], %[sv], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : + [su] "+r"(src_u), [sv] "+r"(src_v), [dst] "+r"(dst_uv), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} + +void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb, + uint8_t* dst_a, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickod.b $vr0, $vr1, $vr0\n\t" + "vpickod.b $vr0, $vr0, $vr0\n\t" + "vstelm.d $vr0, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_a), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); } void I422ToYUY2Row_LSX(const uint8_t* src_y, @@ -348,24 +212,26 @@ void I422ToYUY2Row_LSX(const uint8_t* src_y, const uint8_t* src_v, uint8_t* dst_yuy2, int width) { - int x; - int len = width / 16; - __m128i src_u0, src_v0, src_y0, vec_uv0; - __m128i vec_yuy2_0, vec_yuy2_1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_u, 0, src_v, 0, src_u0, src_v0); - src_y0 = __lsx_vld(src_y, 0); - vec_uv0 = __lsx_vilvl_b(src_v0, src_u0); - vec_yuy2_0 = __lsx_vilvl_b(vec_uv0, src_y0); - vec_yuy2_1 = __lsx_vilvh_b(vec_uv0, src_y0); - __lsx_vst(vec_yuy2_0, dst_yuy2, 0); - __lsx_vst(vec_yuy2_1, dst_yuy2, 16); - src_u += 8; - src_v += 8; - src_y += 16; - dst_yuy2 += 32; - } + asm volatile( + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[su], 0\n\t" + "vldrepl.d $vr2, %[sv], 0\n\t" + "vilvl.b $vr3, $vr2, $vr1\n\t" + "vilvl.b $vr4, $vr3, $vr0\n\t" + "vilvh.b $vr5, $vr3, $vr0\n\t" + "vst $vr4, %[dst], 0\n\t" + "vst $vr5, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_yuy2), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); } void I422ToUYVYRow_LSX(const uint8_t* src_y, @@ -373,24 +239,168 @@ void I422ToUYVYRow_LSX(const uint8_t* src_y, const uint8_t* src_v, uint8_t* dst_uyvy, int width) { - int x; - int len = width / 16; - __m128i src_u0, src_v0, src_y0, vec_uv0; - __m128i vec_uyvy0, vec_uyvy1; + asm volatile( + "1:\n\t" + "vld $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[su], 0\n\t" + "vldrepl.d $vr2, %[sv], 0\n\t" + "vilvl.b $vr3, $vr2, $vr1\n\t" + "vilvl.b $vr4, $vr0, $vr3\n\t" + "vilvh.b $vr5, $vr0, $vr3\n\t" + "vst $vr4, %[dst], 0\n\t" + "vst $vr5, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_uyvy), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); +} - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_u, 0, src_v, 0, src_u0, src_v0); - src_y0 = __lsx_vld(src_y, 0); - vec_uv0 = __lsx_vilvl_b(src_v0, src_u0); - vec_uyvy0 = __lsx_vilvl_b(src_y0, vec_uv0); - vec_uyvy1 = __lsx_vilvh_b(src_y0, vec_uv0); - __lsx_vst(vec_uyvy0, dst_uyvy, 0); - __lsx_vst(vec_uyvy1, dst_uyvy, 16); - src_u += 8; - src_v += 8; - src_y += 16; - dst_uyvy += 32; - } +void YUY2ToYRow_LSX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2"); +} + +void UYVYToYRow_LSX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickod.b $vr2, $vr1, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2"); +} + +void YUY2ToUV422Row_LSX(const uint8_t* src_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickod.b $vr2, $vr1, $vr0\n\t" + "vpickev.b $vr3, $vr2, $vr2\n\t" + "vpickod.b $vr4, $vr2, $vr2\n\t" + "vstelm.d $vr3, %[du], 0, 0\n\t" + "vstelm.d $vr4, %[dv], 0, 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} + +void UYVYToUV422Row_LSX(const uint8_t* src_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickev.b $vr3, $vr2, $vr2\n\t" + "vpickod.b $vr4, $vr2, $vr2\n\t" + "vstelm.d $vr3, %[du], 0, 0\n\t" + "vstelm.d $vr4, %[dv], 0, 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); +} + +void YUY2ToUVRow_LSX(const uint8_t* src_yuy2, + int src_stride_yuy2, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_yuy2 = src_yuy2 + src_stride_yuy2; + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s0], 16\n\t" + "vld $vr2, %[s1], 0\n\t" + "vld $vr3, %[s1], 16\n\t" + "vpickod.b $vr0, $vr1, $vr0\n\t" + "vpickod.b $vr2, $vr3, $vr2\n\t" + "vavgr.bu $vr0, $vr0, $vr2\n\t" + "vpickev.b $vr1, $vr0, $vr0\n\t" + "vpickod.b $vr2, $vr0, $vr0\n\t" + "vstelm.d $vr1, %[du], 0, 0\n\t" + "vstelm.d $vr2, %[dv], 0, 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_yuy2), [s1] "+r"(next_yuy2), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} + +void UYVYToUVRow_LSX(const uint8_t* src_uyvy, + int src_stride_uyvy, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_uyvy = src_uyvy + src_stride_uyvy; + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s0], 16\n\t" + "vld $vr2, %[s1], 0\n\t" + "vld $vr3, %[s1], 16\n\t" + "vpickev.b $vr0, $vr1, $vr0\n\t" + "vpickev.b $vr2, $vr3, $vr2\n\t" + "vavgr.bu $vr0, $vr0, $vr2\n\t" + "vpickev.b $vr1, $vr0, $vr0\n\t" + "vpickod.b $vr2, $vr0, $vr0\n\t" + "vstelm.d $vr1, %[du], 0, 0\n\t" + "vstelm.d $vr2, %[dv], 0, 0\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_uyvy), [s1] "+r"(next_uyvy), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); } void I422ToARGBRow_LSX(const uint8_t* src_y, @@ -399,58 +409,106 @@ void I422ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m128i vec_yb, vec_yg, vec_ub, vec_ug, vec_vr, vec_vg; - __m128i vec_ubvr, vec_ugvg; - __m128i alpha = __lsx_vldi(0xFF); - __m128i const_80 = __lsx_vldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ubvr = __lsx_vilvl_h(vec_ub, vec_vr); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m128i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - STOREARGB_D(alpha, alpha, r_l, r_h, g_l, g_h, b_l, b_h, dst_argb); - src_y += 16; - src_u += 8; - src_v += 8; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", + "$vr23", "$vr24"); } void I422ToRGBARow_LSX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, - uint8_t* dst_rgba, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m128i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m128i vec_ubvr, vec_ugvg; - __m128i alpha = __lsx_vldi(0xFF); - __m128i const_80 = __lsx_vldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ubvr = __lsx_vilvl_h(vec_ub, vec_vr); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m128i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - STOREARGB_D(r_l, r_h, g_l, g_h, b_l, b_h, alpha, alpha, dst_rgba); - src_y += 16; - src_u += 8; - src_v += 8; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr2, $vr5\n\t" + "vilvl.b $vr7, $vr4, $vr3\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", + "$vr23", "$vr24"); } void I422AlphaToARGBRow_LSX(const uint8_t* src_y, @@ -460,41 +518,53 @@ void I422AlphaToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - int res = width & 15; - __m128i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m128i vec_ubvr, vec_ugvg; - __m128i zero = __lsx_vldi(0); - __m128i const_80 = __lsx_vldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ubvr = __lsx_vilvl_h(vec_ub, vec_vr); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m128i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h, a_l, a_h; - - y = __lsx_vld(src_a, 0); - a_l = __lsx_vilvl_b(zero, y); - a_h = __lsx_vilvh_b(zero, y); - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - STOREARGB_D(a_l, a_h, r_l, r_h, g_l, g_h, b_l, b_h, dst_argb); - src_y += 16; - src_u += 8; - src_v += 8; - src_a += 16; - } - if (res) { - __m128i y, uv, r, g, b, a; - a = __lsx_vld(src_a, 0); - a = __lsx_vsllwil_hu_bu(a, 0); - READYUV422(src_y, src_u, src_v, y, uv); - YUVTORGB(y, uv, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b, g, r); - STOREARGB(a, r, g, b, dst_argb); - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vldrepl.d $vr5, %[sa], 0\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[sa], %[sa], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width), [sa] "+r"(src_a) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", + "$vr23", "$vr24"); } void I422ToRGB24Row_LSX(const uint8_t* src_y, @@ -502,1206 +572,256 @@ void I422ToRGB24Row_LSX(const uint8_t* src_y, const uint8_t* src_v, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, - int32_t width) { - int x; - int len = width / 16; - __m128i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m128i vec_ubvr, vec_ugvg; - __m128i const_80 = __lsx_vldi(0x80); - __m128i shuffler0 = {0x0504120302100100, 0x0A18090816070614}; - __m128i shuffler1 = {0x1E0F0E1C0D0C1A0B, 0x1E0F0E1C0D0C1A0B}; - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ubvr = __lsx_vilvl_h(vec_ub, vec_vr); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m128i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - __m128i temp0, temp1, temp2, temp3; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - temp0 = __lsx_vpackev_b(g_l, b_l); - temp1 = __lsx_vpackev_b(g_h, b_h); - DUP4_ARG3(__lsx_vshuf_b, r_l, temp0, shuffler1, r_h, temp1, shuffler1, r_l, - temp0, shuffler0, r_h, temp1, shuffler0, temp2, temp3, temp0, - temp1); - - b_l = __lsx_vilvl_d(temp1, temp2); - b_h = __lsx_vilvh_d(temp3, temp1); - __lsx_vst(temp0, dst_argb, 0); - __lsx_vst(b_l, dst_argb, 16); - __lsx_vst(b_h, dst_argb, 32); - dst_argb += 48; - src_y += 16; - src_u += 8; - src_v += 8; - } + int width) { + static const uint8_t kShuf0[16] = {0, 1, 2, 4, 5, 6, 8, 9, + 10, 12, 13, 14, 16, 17, 18, 20}; + static const uint8_t kShuf1[16] = {21, 22, 24, 25, 26, 28, 29, 30, + 0, 0, 0, 0, 0, 0, 0, 0}; + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vld $vr12, %[s0], 0\n\t" + "vld $vr13, %[s1], 0\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr4, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vshuf.b $vr10, $vr9, $vr8, $vr12\n\t" + "vshuf.b $vr11, $vr9, $vr8, $vr13\n\t" + "vst $vr10, %[dst], 0\n\t" + "vstelm.d $vr11, %[dst], 16, 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 24\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants), [s0] "r"(kShuf0), [s1] "r"(kShuf1) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7", + "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr20", "$vr21", + "$vr22", "$vr23", "$vr24"); } -// TODO(fbarchard): Consider AND instead of shift to isolate 5 upper bits of R. void I422ToRGB565Row_LSX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, - uint8_t* dst_rgb565, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m128i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m128i vec_ubvr, vec_ugvg; - __m128i const_80 = __lsx_vldi(0x80); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ubvr = __lsx_vilvl_h(vec_ub, vec_vr); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m128i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - b_l = __lsx_vsrli_h(b_l, 3); - b_h = __lsx_vsrli_h(b_h, 3); - g_l = __lsx_vsrli_h(g_l, 2); - g_h = __lsx_vsrli_h(g_h, 2); - r_l = __lsx_vsrli_h(r_l, 3); - r_h = __lsx_vsrli_h(r_h, 3); - r_l = __lsx_vslli_h(r_l, 11); - r_h = __lsx_vslli_h(r_h, 11); - g_l = __lsx_vslli_h(g_l, 5); - g_h = __lsx_vslli_h(g_h, 5); - r_l = __lsx_vor_v(r_l, g_l); - r_l = __lsx_vor_v(r_l, b_l); - r_h = __lsx_vor_v(r_h, g_h); - r_h = __lsx_vor_v(r_h, b_h); - __lsx_vst(r_l, dst_rgb565, 0); - __lsx_vst(r_h, dst_rgb565, 16); - dst_rgb565 += 32; - src_y += 16; - src_u += 8; - src_v += 8; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr14, 31\n\t" + "vsrli.h $vr14, $vr14, 8\n\t" + "vldi $vr15, 63\n\t" + "vsrli.h $vr15, $vr15, 8\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 2\n\t" + "vsrli.b $vr4, $vr4, 3\n\t" + "vilvl.b $vr2, $vr2, $vr2\n\t" + "vilvl.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr4, $vr4, $vr4\n\t" + "vand.v $vr2, $vr2, $vr14\n\t" + "vand.v $vr3, $vr3, $vr15\n\t" + "vand.v $vr4, $vr4, $vr14\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr4, $vr4, 11\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", + "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); } -// TODO(fbarchard): Consider AND instead of shift to isolate 4 upper bits of G. void I422ToARGB4444Row_LSX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, - uint8_t* dst_argb4444, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m128i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m128i vec_ubvr, vec_ugvg; - __m128i const_80 = __lsx_vldi(0x80); - __m128i alpha = (__m128i)v2u64{0xF000F000F000F000, 0xF000F000F000F000}; - __m128i mask = {0x00F000F000F000F0, 0x00F000F000F000F0}; - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ubvr = __lsx_vilvl_h(vec_ub, vec_vr); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m128i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - b_l = __lsx_vsrli_h(b_l, 4); - b_h = __lsx_vsrli_h(b_h, 4); - r_l = __lsx_vsrli_h(r_l, 4); - r_h = __lsx_vsrli_h(r_h, 4); - g_l = __lsx_vand_v(g_l, mask); - g_h = __lsx_vand_v(g_h, mask); - r_l = __lsx_vslli_h(r_l, 8); - r_h = __lsx_vslli_h(r_h, 8); - r_l = __lsx_vor_v(r_l, alpha); - r_h = __lsx_vor_v(r_h, alpha); - r_l = __lsx_vor_v(r_l, g_l); - r_h = __lsx_vor_v(r_h, g_h); - r_l = __lsx_vor_v(r_l, b_l); - r_h = __lsx_vor_v(r_h, b_h); - __lsx_vst(r_l, dst_argb4444, 0); - __lsx_vst(r_h, dst_argb4444, 16); - dst_argb4444 += 32; - src_y += 16; - src_u += 8; - src_v += 8; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr14, 15\n\t" + "vsrli.h $vr14, $vr14, 8\n\t" + "vldi $vr15, 240\n\t" + "vsrli.h $vr15, $vr15, 8\n\t" + "vslli.h $vr15, $vr15, 8\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vsrli.b $vr2, $vr2, 4\n\t" + "vsrli.b $vr3, $vr3, 4\n\t" + "vsrli.b $vr4, $vr4, 4\n\t" + "vilvl.b $vr2, $vr2, $vr2\n\t" + "vilvl.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr4, $vr4, $vr4\n\t" + "vand.v $vr2, $vr2, $vr14\n\t" + "vand.v $vr3, $vr3, $vr14\n\t" + "vand.v $vr4, $vr4, $vr14\n\t" + "vslli.h $vr3, $vr3, 4\n\t" + "vslli.h $vr4, $vr4, 8\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vor.v $vr2, $vr2, $vr15\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", + "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); } void I422ToARGB1555Row_LSX(const uint8_t* src_y, const uint8_t* src_u, const uint8_t* src_v, - uint8_t* dst_argb1555, + uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m128i vec_yb, vec_yg, vec_ub, vec_vr, vec_ug, vec_vg; - __m128i vec_ubvr, vec_ugvg; - __m128i const_80 = __lsx_vldi(0x80); - __m128i alpha = (__m128i)v2u64{0x8000800080008000, 0x8000800080008000}; - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ubvr = __lsx_vilvl_h(vec_ub, vec_vr); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - __m128i y, uv_l, uv_h, b_l, b_h, g_l, g_h, r_l, r_h; - - READYUV422_D(src_y, src_u, src_v, y, uv_l, uv_h); - YUVTORGB_D(y, uv_l, uv_h, vec_ubvr, vec_ugvg, vec_yg, vec_yb, b_l, b_h, g_l, - g_h, r_l, r_h); - b_l = __lsx_vsrli_h(b_l, 3); - b_h = __lsx_vsrli_h(b_h, 3); - g_l = __lsx_vsrli_h(g_l, 3); - - g_h = __lsx_vsrli_h(g_h, 3); - g_l = __lsx_vslli_h(g_l, 5); - g_h = __lsx_vslli_h(g_h, 5); - r_l = __lsx_vsrli_h(r_l, 3); - r_h = __lsx_vsrli_h(r_h, 3); - r_l = __lsx_vslli_h(r_l, 10); - r_h = __lsx_vslli_h(r_h, 10); - r_l = __lsx_vor_v(r_l, alpha); - r_h = __lsx_vor_v(r_h, alpha); - r_l = __lsx_vor_v(r_l, g_l); - r_h = __lsx_vor_v(r_h, g_h); - r_l = __lsx_vor_v(r_l, b_l); - r_h = __lsx_vor_v(r_h, b_h); - __lsx_vst(r_l, dst_argb1555, 0); - __lsx_vst(r_h, dst_argb1555, 16); - dst_argb1555 += 32; - src_y += 16; - src_u += 8; - src_v += 8; - } -} - -void YUY2ToYRow_LSX(const uint8_t* src_yuy2, uint8_t* dst_y, int width) { - int x; - int len = width / 16; - __m128i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_yuy2, 0, src_yuy2, 16, src0, src1); - dst0 = __lsx_vpickev_b(src1, src0); - __lsx_vst(dst0, dst_y, 0); - src_yuy2 += 32; - dst_y += 16; - } -} - -void YUY2ToUVRow_LSX(const uint8_t* src_yuy2, - int src_stride_yuy2, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - const uint8_t* src_yuy2_next = src_yuy2 + src_stride_yuy2; - int x; - int len = width / 16; - __m128i src0, src1, src2, src3; - __m128i tmp0, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_yuy2, 0, src_yuy2, 16, src_yuy2_next, 0, - src_yuy2_next, 16, src0, src1, src2, src3); - src0 = __lsx_vpickod_b(src1, src0); - src1 = __lsx_vpickod_b(src3, src2); - tmp0 = __lsx_vavgr_bu(src1, src0); - dst0 = __lsx_vpickev_b(tmp0, tmp0); - dst1 = __lsx_vpickod_b(tmp0, tmp0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst1, dst_v, 0, 0); - src_yuy2 += 32; - src_yuy2_next += 32; - dst_u += 8; - dst_v += 8; - } -} - -void YUY2ToUV422Row_LSX(const uint8_t* src_yuy2, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 16; - __m128i src0, src1, tmp0, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_yuy2, 0, src_yuy2, 16, src0, src1); - tmp0 = __lsx_vpickod_b(src1, src0); - dst0 = __lsx_vpickev_b(tmp0, tmp0); - dst1 = __lsx_vpickod_b(tmp0, tmp0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst1, dst_v, 0, 0); - src_yuy2 += 32; - dst_u += 8; - dst_v += 8; - } -} - -void UYVYToYRow_LSX(const uint8_t* src_uyvy, uint8_t* dst_y, int width) { - int x; - int len = width / 16; - __m128i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_uyvy, 0, src_uyvy, 16, src0, src1); - dst0 = __lsx_vpickod_b(src1, src0); - __lsx_vst(dst0, dst_y, 0); - src_uyvy += 32; - dst_y += 16; - } -} - -void UYVYToUVRow_LSX(const uint8_t* src_uyvy, - int src_stride_uyvy, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - const uint8_t* src_uyvy_next = src_uyvy + src_stride_uyvy; - int x; - int len = width / 16; - __m128i src0, src1, src2, src3, tmp0, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_uyvy, 0, src_uyvy, 16, src_uyvy_next, 0, - src_uyvy_next, 16, src0, src1, src2, src3); - src0 = __lsx_vpickev_b(src1, src0); - src1 = __lsx_vpickev_b(src3, src2); - tmp0 = __lsx_vavgr_bu(src1, src0); - dst0 = __lsx_vpickev_b(tmp0, tmp0); - dst1 = __lsx_vpickod_b(tmp0, tmp0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst1, dst_v, 0, 0); - src_uyvy += 32; - src_uyvy_next += 32; - dst_u += 8; - dst_v += 8; - } -} - -void UYVYToUV422Row_LSX(const uint8_t* src_uyvy, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 16; - __m128i src0, src1, tmp0, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_uyvy, 0, src_uyvy, 16, src0, src1); - tmp0 = __lsx_vpickev_b(src1, src0); - dst0 = __lsx_vpickev_b(tmp0, tmp0); - dst1 = __lsx_vpickod_b(tmp0, tmp0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst1, dst_v, 0, 0); - src_uyvy += 32; - dst_u += 8; - dst_v += 8; - } -} -void ARGBToRGB24Row_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - int x; - int len = (width / 16) - 1; - __m128i src0, src1, src2, src3; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i shuf = {0x0908060504020100, 0x000000000E0D0C0A}; - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src_argb, 32, src_argb, 48, - src0, src1, src2, src3); - tmp0 = __lsx_vshuf_b(src0, src0, shuf); - tmp1 = __lsx_vshuf_b(src1, src1, shuf); - tmp2 = __lsx_vshuf_b(src2, src2, shuf); - tmp3 = __lsx_vshuf_b(src3, src3, shuf); - __lsx_vst(tmp0, dst_rgb, 0); - __lsx_vst(tmp1, dst_rgb, 12); - __lsx_vst(tmp2, dst_rgb, 24); - __lsx_vst(tmp3, dst_rgb, 36); - dst_rgb += 48; - src_argb += 64; - } - DUP4_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src_argb, 32, src_argb, 48, - src0, src1, src2, src3); - tmp0 = __lsx_vshuf_b(src0, src0, shuf); - tmp1 = __lsx_vshuf_b(src1, src1, shuf); - tmp2 = __lsx_vshuf_b(src2, src2, shuf); - tmp3 = __lsx_vshuf_b(src3, src3, shuf); - __lsx_vst(tmp0, dst_rgb, 0); - __lsx_vst(tmp1, dst_rgb, 12); - __lsx_vst(tmp2, dst_rgb, 24); - dst_rgb += 36; - __lsx_vst(tmp3, dst_rgb, 0); -} - -void ARGBToRAWRow_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - int x; - int len = (width / 16) - 1; - __m128i src0, src1, src2, src3; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i shuf = {0x090A040506000102, 0x000000000C0D0E08}; - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src_argb, 32, src_argb, 48, - src0, src1, src2, src3); - tmp0 = __lsx_vshuf_b(src0, src0, shuf); - tmp1 = __lsx_vshuf_b(src1, src1, shuf); - tmp2 = __lsx_vshuf_b(src2, src2, shuf); - tmp3 = __lsx_vshuf_b(src3, src3, shuf); - __lsx_vst(tmp0, dst_rgb, 0); - __lsx_vst(tmp1, dst_rgb, 12); - __lsx_vst(tmp2, dst_rgb, 24); - __lsx_vst(tmp3, dst_rgb, 36); - dst_rgb += 48; - src_argb += 64; - } - DUP4_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src_argb, 32, src_argb, 48, - src0, src1, src2, src3); - tmp0 = __lsx_vshuf_b(src0, src0, shuf); - tmp1 = __lsx_vshuf_b(src1, src1, shuf); - tmp2 = __lsx_vshuf_b(src2, src2, shuf); - tmp3 = __lsx_vshuf_b(src3, src3, shuf); - __lsx_vst(tmp0, dst_rgb, 0); - __lsx_vst(tmp1, dst_rgb, 12); - __lsx_vst(tmp2, dst_rgb, 24); - dst_rgb += 36; - __lsx_vst(tmp3, dst_rgb, 0); -} - -void ARGBToRGB565Row_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { - int x; - int len = width / 8; - __m128i zero = __lsx_vldi(0); - __m128i src0, src1, tmp0, tmp1, dst0; - __m128i shift = {0x0300030003000300, 0x0300030003000300}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - tmp0 = __lsx_vsrli_b(tmp0, 3); - tmp1 = __lsx_vpackev_b(zero, tmp1); - tmp1 = __lsx_vsrli_h(tmp1, 2); - tmp0 = __lsx_vsll_b(tmp0, shift); - tmp1 = __lsx_vslli_h(tmp1, 5); - dst0 = __lsx_vor_v(tmp0, tmp1); - __lsx_vst(dst0, dst_rgb, 0); - dst_rgb += 16; - src_argb += 32; - } -} - -void ARGBToARGB1555Row_LSX(const uint8_t* src_argb, - uint8_t* dst_rgb, - int width) { - int x; - int len = width / 8; - __m128i zero = __lsx_vldi(0); - __m128i src0, src1, tmp0, tmp1, tmp2, tmp3, dst0; - __m128i shift1 = {0x0703070307030703, 0x0703070307030703}; - __m128i shift2 = {0x0200020002000200, 0x0200020002000200}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - tmp0 = __lsx_vsrli_b(tmp0, 3); - tmp1 = __lsx_vsrl_b(tmp1, shift1); - tmp0 = __lsx_vsll_b(tmp0, shift2); - tmp2 = __lsx_vpackev_b(zero, tmp1); - tmp3 = __lsx_vpackod_b(zero, tmp1); - tmp2 = __lsx_vslli_h(tmp2, 5); - tmp3 = __lsx_vslli_h(tmp3, 15); - dst0 = __lsx_vor_v(tmp0, tmp2); - dst0 = __lsx_vor_v(dst0, tmp3); - __lsx_vst(dst0, dst_rgb, 0); - dst_rgb += 16; - src_argb += 32; - } -} - -void ARGBToARGB4444Row_LSX(const uint8_t* src_argb, - uint8_t* dst_rgb, - int width) { - int x; - int len = width / 8; - __m128i src0, src1, tmp0, tmp1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - tmp1 = __lsx_vandi_b(tmp1, 0xF0); - tmp0 = __lsx_vsrli_b(tmp0, 4); - dst0 = __lsx_vor_v(tmp1, tmp0); - __lsx_vst(dst0, dst_rgb, 0); - dst_rgb += 16; - src_argb += 32; - } -} - -void ARGBMultiplyRow_LSX(const uint8_t* src_argb0, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 4; - __m128i half = __lsx_vldi(0x480); - __m128i src0, src1, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb0, 0, src_argb1, 0, src0, src1); - dst0 = __lsx_vmaddwev_h_bu(half, src0, src1); - dst1 = __lsx_vmaddwod_h_bu(half, src0, src1); - dst0 = __lsx_vpackod_b(dst1, dst0); - __lsx_vst(dst0, dst_argb, 0); - src_argb0 += 16; - src_argb1 += 16; - dst_argb += 16; - } -} - -void ARGBAddRow_LSX(const uint8_t* src_argb0, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 4; - __m128i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb0, 0, src_argb1, 0, src0, src1); - dst0 = __lsx_vsadd_bu(src0, src1); - __lsx_vst(dst0, dst_argb, 0); - src_argb0 += 16; - src_argb1 += 16; - dst_argb += 16; - } -} - -void ARGBSubtractRow_LSX(const uint8_t* src_argb0, - const uint8_t* src_argb1, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 4; - __m128i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb0, 0, src_argb1, 0, src0, src1); - dst0 = __lsx_vssub_bu(src0, src1); - __lsx_vst(dst0, dst_argb, 0); - src_argb0 += 16; - src_argb1 += 16; - dst_argb += 16; - } -} - -void ARGBAttenuateRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 8; - __m128i src0, src1, tmp0, tmp1; - __m128i reg0, reg1, reg2, reg3, reg4, reg5; - __m128i b, g, r, a, dst0, dst1; - __m128i control = {0x0005000100040000, 0x0007000300060002}; - __m128i zero = __lsx_vldi(0); - __m128i const_add = __lsx_vldi(0x8ff); - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - b = __lsx_vpackev_b(zero, tmp0); - r = __lsx_vpackod_b(zero, tmp0); - g = __lsx_vpackev_b(zero, tmp1); - a = __lsx_vpackod_b(zero, tmp1); - reg0 = __lsx_vmaddwev_w_hu(const_add, b, a); - reg1 = __lsx_vmaddwod_w_hu(const_add, b, a); - reg2 = __lsx_vmaddwev_w_hu(const_add, r, a); - reg3 = __lsx_vmaddwod_w_hu(const_add, r, a); - reg4 = __lsx_vmaddwev_w_hu(const_add, g, a); - reg5 = __lsx_vmaddwod_w_hu(const_add, g, a); - reg0 = __lsx_vssrani_h_w(reg1, reg0, 8); - reg2 = __lsx_vssrani_h_w(reg3, reg2, 8); - reg4 = __lsx_vssrani_h_w(reg5, reg4, 8); - reg0 = __lsx_vshuf_h(control, reg0, reg0); - reg2 = __lsx_vshuf_h(control, reg2, reg2); - reg4 = __lsx_vshuf_h(control, reg4, reg4); - tmp0 = __lsx_vpackev_b(reg4, reg0); - tmp1 = __lsx_vpackev_b(a, reg2); - dst0 = __lsx_vilvl_h(tmp1, tmp0); - dst1 = __lsx_vilvh_h(tmp1, tmp0); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - dst_argb += 32; - src_argb += 32; - } -} - -void ARGBToRGB565DitherRow_LSX(const uint8_t* src_argb, - uint8_t* dst_rgb, - uint32_t dither4, - int width) { - int x; - int len = width / 8; - __m128i src0, src1, tmp0, tmp1, dst0; - __m128i b, g, r; - __m128i zero = __lsx_vldi(0); - __m128i vec_dither = __lsx_vldrepl_w(&dither4, 0); - - vec_dither = __lsx_vilvl_b(zero, vec_dither); - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - b = __lsx_vpackev_b(zero, tmp0); - r = __lsx_vpackod_b(zero, tmp0); - g = __lsx_vpackev_b(zero, tmp1); - b = __lsx_vadd_h(b, vec_dither); - g = __lsx_vadd_h(g, vec_dither); - r = __lsx_vadd_h(r, vec_dither); - DUP2_ARG1(__lsx_vclip255_h, b, g, b, g); - r = __lsx_vclip255_h(r); - b = __lsx_vsrai_h(b, 3); - g = __lsx_vsrai_h(g, 2); - r = __lsx_vsrai_h(r, 3); - g = __lsx_vslli_h(g, 5); - r = __lsx_vslli_h(r, 11); - dst0 = __lsx_vor_v(b, g); - dst0 = __lsx_vor_v(dst0, r); - __lsx_vst(dst0, dst_rgb, 0); - src_argb += 32; - dst_rgb += 16; - } -} - -void ARGBShuffleRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - const uint8_t* shuffler, - int width) { - int x; - int len = width / 8; - __m128i src0, src1, dst0, dst1; - __m128i shuf = {0x0404040400000000, 0x0C0C0C0C08080808}; - __m128i temp = __lsx_vldrepl_w(shuffler, 0); - - shuf = __lsx_vadd_b(shuf, temp); - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - dst0 = __lsx_vshuf_b(src0, src0, shuf); - dst1 = __lsx_vshuf_b(src1, src1, shuf); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - src_argb += 32; - dst_argb += 32; - } -} - -void ARGBShadeRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - int width, - uint32_t value) { - int x; - int len = width / 4; - __m128i src0, dst0, tmp0, tmp1; - __m128i vec_value = __lsx_vreplgr2vr_w(value); - - vec_value = __lsx_vilvl_b(vec_value, vec_value); - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_argb, 0); - tmp0 = __lsx_vilvl_b(src0, src0); - tmp1 = __lsx_vilvh_b(src0, src0); - tmp0 = __lsx_vmuh_hu(tmp0, vec_value); - tmp1 = __lsx_vmuh_hu(tmp1, vec_value); - dst0 = __lsx_vpickod_b(tmp1, tmp0); - __lsx_vst(dst0, dst_argb, 0); - src_argb += 16; - dst_argb += 16; - } -} - -void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { - int x; - int len = width / 8; - __m128i src0, src1, tmp0, tmp1; - __m128i reg0, reg1, reg2, dst0, dst1; - __m128i const_128 = __lsx_vldi(0x480); - __m128i const_150 = __lsx_vldi(0x96); - __m128i const_br = {0x4D1D4D1D4D1D4D1D, 0x4D1D4D1D4D1D4D1D}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - reg0 = __lsx_vdp2_h_bu(tmp0, const_br); - reg1 = __lsx_vmaddwev_h_bu(const_128, tmp1, const_150); - reg2 = __lsx_vadd_h(reg0, reg1); - tmp0 = __lsx_vpackod_b(reg2, reg2); - tmp1 = __lsx_vpackod_b(tmp1, reg2); - dst0 = __lsx_vilvl_h(tmp1, tmp0); - dst1 = __lsx_vilvh_h(tmp1, tmp0); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - src_argb += 32; - dst_argb += 32; - } -} - -void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width) { - int x; - int len = width / 8; - __m128i src0, src1, tmp0, tmp1; - __m128i reg0, reg1, spb, spg, spr; - __m128i dst0, dst1; - __m128i spb_g = __lsx_vldi(68); - __m128i spg_g = __lsx_vldi(88); - __m128i spr_g = __lsx_vldi(98); - __m128i spb_br = {0x2311231123112311, 0x2311231123112311}; - __m128i spg_br = {0x2D162D162D162D16, 0x2D162D162D162D16}; - __m128i spr_br = {0x3218321832183218, 0x3218321832183218}; - __m128i shuff = {0x1706150413021100, 0x1F0E1D0C1B0A1908}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, dst_argb, 0, dst_argb, 16, src0, src1); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - DUP2_ARG2(__lsx_vdp2_h_bu, tmp0, spb_br, tmp0, spg_br, spb, spg); - spr = __lsx_vdp2_h_bu(tmp0, spr_br); - spb = __lsx_vmaddwev_h_bu(spb, tmp1, spb_g); - spg = __lsx_vmaddwev_h_bu(spg, tmp1, spg_g); - spr = __lsx_vmaddwev_h_bu(spr, tmp1, spr_g); - spb = __lsx_vsrli_h(spb, 7); - spg = __lsx_vsrli_h(spg, 7); - spr = __lsx_vsrli_h(spr, 7); - spg = __lsx_vsat_hu(spg, 7); - spr = __lsx_vsat_hu(spr, 7); - reg0 = __lsx_vpackev_b(spg, spb); - reg1 = __lsx_vshuf_b(tmp1, spr, shuff); - dst0 = __lsx_vilvl_h(reg1, reg0); - dst1 = __lsx_vilvh_h(reg1, reg0); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - dst_argb += 32; - } -} - -void ARGB4444ToARGBRow_LSX(const uint8_t* src_argb4444, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 16; - __m128i src0, src1; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i reg0, reg1, reg2, reg3; - __m128i dst0, dst1, dst2, dst3; - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_argb4444, 0); - src1 = __lsx_vld(src_argb4444, 16); - tmp0 = __lsx_vandi_b(src0, 0x0F); - tmp1 = __lsx_vandi_b(src0, 0xF0); - tmp2 = __lsx_vandi_b(src1, 0x0F); - tmp3 = __lsx_vandi_b(src1, 0xF0); - reg0 = __lsx_vslli_b(tmp0, 4); - reg2 = __lsx_vslli_b(tmp2, 4); - reg1 = __lsx_vsrli_b(tmp1, 4); - reg3 = __lsx_vsrli_b(tmp3, 4); - DUP4_ARG2(__lsx_vor_v, tmp0, reg0, tmp1, reg1, tmp2, reg2, tmp3, reg3, tmp0, - tmp1, tmp2, tmp3); - dst0 = __lsx_vilvl_b(tmp1, tmp0); - dst2 = __lsx_vilvl_b(tmp3, tmp2); - dst1 = __lsx_vilvh_b(tmp1, tmp0); - dst3 = __lsx_vilvh_b(tmp3, tmp2); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - dst_argb += 64; - src_argb4444 += 32; - } -} - -void ARGB1555ToARGBRow_LSX(const uint8_t* src_argb1555, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 16; - __m128i src0, src1; - __m128i tmp0, tmp1, tmpb, tmpg, tmpr, tmpa; - __m128i reg0, reg1, reg2; - __m128i dst0, dst1, dst2, dst3; - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_argb1555, 0); - src1 = __lsx_vld(src_argb1555, 16); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - tmpb = __lsx_vandi_b(tmp0, 0x1F); - tmpg = __lsx_vsrli_b(tmp0, 5); - reg0 = __lsx_vandi_b(tmp1, 0x03); - reg0 = __lsx_vslli_b(reg0, 3); - tmpg = __lsx_vor_v(tmpg, reg0); - reg1 = __lsx_vandi_b(tmp1, 0x7C); - tmpr = __lsx_vsrli_b(reg1, 2); - tmpa = __lsx_vsrli_b(tmp1, 7); - tmpa = __lsx_vneg_b(tmpa); - reg0 = __lsx_vslli_b(tmpb, 3); - reg1 = __lsx_vslli_b(tmpg, 3); - reg2 = __lsx_vslli_b(tmpr, 3); - tmpb = __lsx_vsrli_b(tmpb, 2); - tmpg = __lsx_vsrli_b(tmpg, 2); - tmpr = __lsx_vsrli_b(tmpr, 2); - tmpb = __lsx_vor_v(reg0, tmpb); - tmpg = __lsx_vor_v(reg1, tmpg); - tmpr = __lsx_vor_v(reg2, tmpr); - DUP2_ARG2(__lsx_vilvl_b, tmpg, tmpb, tmpa, tmpr, reg0, reg1); - dst0 = __lsx_vilvl_h(reg1, reg0); - dst1 = __lsx_vilvh_h(reg1, reg0); - DUP2_ARG2(__lsx_vilvh_b, tmpg, tmpb, tmpa, tmpr, reg0, reg1); - dst2 = __lsx_vilvl_h(reg1, reg0); - dst3 = __lsx_vilvh_h(reg1, reg0); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - dst_argb += 64; - src_argb1555 += 32; - } -} - -void RGB565ToARGBRow_LSX(const uint8_t* src_rgb565, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 16; - __m128i src0, src1; - __m128i tmp0, tmp1, tmpb, tmpg, tmpr; - __m128i reg0, reg1, dst0, dst1, dst2, dst3; - __m128i alpha = __lsx_vldi(0xFF); - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_rgb565, 0); - src1 = __lsx_vld(src_rgb565, 16); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - tmpb = __lsx_vandi_b(tmp0, 0x1F); - tmpr = __lsx_vandi_b(tmp1, 0xF8); - reg1 = __lsx_vandi_b(tmp1, 0x07); - reg0 = __lsx_vsrli_b(tmp0, 5); - reg1 = __lsx_vslli_b(reg1, 3); - tmpg = __lsx_vor_v(reg1, reg0); - reg0 = __lsx_vslli_b(tmpb, 3); - reg1 = __lsx_vsrli_b(tmpb, 2); - tmpb = __lsx_vor_v(reg1, reg0); - reg0 = __lsx_vslli_b(tmpg, 2); - reg1 = __lsx_vsrli_b(tmpg, 4); - tmpg = __lsx_vor_v(reg1, reg0); - reg0 = __lsx_vsrli_b(tmpr, 5); - tmpr = __lsx_vor_v(tmpr, reg0); - DUP2_ARG2(__lsx_vilvl_b, tmpg, tmpb, alpha, tmpr, reg0, reg1); - dst0 = __lsx_vilvl_h(reg1, reg0); - dst1 = __lsx_vilvh_h(reg1, reg0); - DUP2_ARG2(__lsx_vilvh_b, tmpg, tmpb, alpha, tmpr, reg0, reg1); - dst2 = __lsx_vilvl_h(reg1, reg0); - dst3 = __lsx_vilvh_h(reg1, reg0); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - dst_argb += 64; - src_rgb565 += 32; - } -} - -void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 16; - __m128i src0, src1, src2; - __m128i tmp0, tmp1, tmp2; - __m128i dst0, dst1, dst2, dst3; - __m128i alpha = __lsx_vldi(0xFF); - __m128i shuf0 = {0x131211100F0E0D0C, 0x1B1A191817161514}; - __m128i shuf1 = {0x1F1E1D1C1B1A1918, 0x0706050403020100}; - __m128i shuf2 = {0x0B0A090807060504, 0x131211100F0E0D0C}; - __m128i shuf3 = {0x1005040310020100, 0x100B0A0910080706}; - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_rgb24, 0); - src1 = __lsx_vld(src_rgb24, 16); - src2 = __lsx_vld(src_rgb24, 32); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuf0, src1, src2, shuf1, tmp0, tmp1); - tmp2 = __lsx_vshuf_b(src1, src2, shuf2); - DUP4_ARG3(__lsx_vshuf_b, alpha, src0, shuf3, alpha, tmp0, shuf3, alpha, - tmp1, shuf3, alpha, tmp2, shuf3, dst0, dst1, dst2, dst3); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - dst_argb += 64; - src_rgb24 += 48; - } -} - -void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { - int x; - int len = width / 16; - __m128i src0, src1, src2; - __m128i tmp0, tmp1, tmp2; - __m128i dst0, dst1, dst2, dst3; - __m128i alpha = __lsx_vldi(0xFF); - __m128i shuf0 = {0x131211100F0E0D0C, 0x1B1A191817161514}; - __m128i shuf1 = {0x1F1E1D1C1B1A1918, 0x0706050403020100}; - __m128i shuf2 = {0x0B0A090807060504, 0x131211100F0E0D0C}; - __m128i shuf3 = {0x1003040510000102, 0x10090A0B10060708}; - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_raw, 0); - src1 = __lsx_vld(src_raw, 16); - src2 = __lsx_vld(src_raw, 32); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuf0, src1, src2, shuf1, tmp0, tmp1); - tmp2 = __lsx_vshuf_b(src1, src2, shuf2); - DUP4_ARG3(__lsx_vshuf_b, alpha, src0, shuf3, alpha, tmp0, shuf3, alpha, - tmp1, shuf3, alpha, tmp2, shuf3, dst0, dst1, dst2, dst3); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - dst_argb += 64; - src_raw += 48; - } -} - -void ARGB1555ToYRow_LSX(const uint8_t* src_argb1555, - uint8_t* dst_y, - int width) { - int x; - int len = width / 16; - __m128i src0, src1; - __m128i tmp0, tmp1, tmpb, tmpg, tmpr; - __m128i reg0, reg1, reg2, dst0; - __m128i const_66 = __lsx_vldi(66); - __m128i const_129 = __lsx_vldi(129); - __m128i const_25 = __lsx_vldi(25); - __m128i const_1080 = {0x1080108010801080, 0x1080108010801080}; - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_argb1555, 0); - src1 = __lsx_vld(src_argb1555, 16); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - tmpb = __lsx_vandi_b(tmp0, 0x1F); - tmpg = __lsx_vsrli_b(tmp0, 5); - reg0 = __lsx_vandi_b(tmp1, 0x03); - reg0 = __lsx_vslli_b(reg0, 3); - tmpg = __lsx_vor_v(tmpg, reg0); - reg1 = __lsx_vandi_b(tmp1, 0x7C); - tmpr = __lsx_vsrli_b(reg1, 2); - reg0 = __lsx_vslli_b(tmpb, 3); - reg1 = __lsx_vslli_b(tmpg, 3); - reg2 = __lsx_vslli_b(tmpr, 3); - tmpb = __lsx_vsrli_b(tmpb, 2); - tmpg = __lsx_vsrli_b(tmpg, 2); - tmpr = __lsx_vsrli_b(tmpr, 2); - tmpb = __lsx_vor_v(reg0, tmpb); - tmpg = __lsx_vor_v(reg1, tmpg); - tmpr = __lsx_vor_v(reg2, tmpr); - reg0 = __lsx_vmaddwev_h_bu(const_1080, tmpb, const_25); - reg1 = __lsx_vmaddwod_h_bu(const_1080, tmpb, const_25); - reg0 = __lsx_vmaddwev_h_bu(reg0, tmpg, const_129); - reg1 = __lsx_vmaddwod_h_bu(reg1, tmpg, const_129); - reg0 = __lsx_vmaddwev_h_bu(reg0, tmpr, const_66); - reg1 = __lsx_vmaddwod_h_bu(reg1, tmpr, const_66); - dst0 = __lsx_vpackod_b(reg1, reg0); - __lsx_vst(dst0, dst_y, 0); - dst_y += 16; - src_argb1555 += 32; - } -} - -void ARGB1555ToUVRow_LSX(const uint8_t* src_argb1555, - int src_stride_argb1555, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 16; - const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555; - __m128i src0, src1, src2, src3; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m128i reg0, reg1, reg2, reg3, dst0; - __m128i const_112 = __lsx_vldi(0x470); - __m128i const_74 = __lsx_vldi(0x44A); - __m128i const_38 = __lsx_vldi(0x426); - __m128i const_94 = __lsx_vldi(0x45E); - __m128i const_18 = __lsx_vldi(0x412); - __m128i const_8000 = (__m128i)v2u64{0x8000800080008000, 0x8000800080008000}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_argb1555, 0, src_argb1555, 16, next_argb1555, 0, - next_argb1555, 16, src0, src1, src2, src3); - DUP2_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, tmp0, tmp2); - DUP2_ARG2(__lsx_vpickod_b, src1, src0, src3, src2, tmp1, tmp3); - tmpb = __lsx_vandi_b(tmp0, 0x1F); - nexb = __lsx_vandi_b(tmp2, 0x1F); - tmpg = __lsx_vsrli_b(tmp0, 5); - nexg = __lsx_vsrli_b(tmp2, 5); - reg0 = __lsx_vandi_b(tmp1, 0x03); - reg2 = __lsx_vandi_b(tmp3, 0x03); - reg0 = __lsx_vslli_b(reg0, 3); - reg2 = __lsx_vslli_b(reg2, 3); - tmpg = __lsx_vor_v(tmpg, reg0); - nexg = __lsx_vor_v(nexg, reg2); - reg1 = __lsx_vandi_b(tmp1, 0x7C); - reg3 = __lsx_vandi_b(tmp3, 0x7C); - tmpr = __lsx_vsrli_b(reg1, 2); - nexr = __lsx_vsrli_b(reg3, 2); - reg0 = __lsx_vslli_b(tmpb, 3); - reg1 = __lsx_vslli_b(tmpg, 3); - reg2 = __lsx_vslli_b(tmpr, 3); - tmpb = __lsx_vsrli_b(tmpb, 2); - tmpg = __lsx_vsrli_b(tmpg, 2); - tmpr = __lsx_vsrli_b(tmpr, 2); - tmpb = __lsx_vor_v(reg0, tmpb); - tmpg = __lsx_vor_v(reg1, tmpg); - tmpr = __lsx_vor_v(reg2, tmpr); - reg0 = __lsx_vslli_b(nexb, 3); - reg1 = __lsx_vslli_b(nexg, 3); - reg2 = __lsx_vslli_b(nexr, 3); - nexb = __lsx_vsrli_b(nexb, 2); - nexg = __lsx_vsrli_b(nexg, 2); - nexr = __lsx_vsrli_b(nexr, 2); - nexb = __lsx_vor_v(reg0, nexb); - nexg = __lsx_vor_v(reg1, nexg); - nexr = __lsx_vor_v(reg2, nexr); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, dst0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst0, dst_v, 0, 1); - dst_u += 8; - dst_v += 8; - src_argb1555 += 32; - next_argb1555 += 32; - } -} - -void RGB565ToYRow_LSX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { - int x; - int len = width / 16; - __m128i src0, src1; - __m128i tmp0, tmp1, tmpb, tmpg, tmpr; - __m128i reg0, reg1, dst0; - __m128i const_66 = __lsx_vldi(66); - __m128i const_129 = __lsx_vldi(129); - __m128i const_25 = __lsx_vldi(25); - __m128i const_1080 = {0x1080108010801080, 0x1080108010801080}; - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_rgb565, 0); - src1 = __lsx_vld(src_rgb565, 16); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - tmpb = __lsx_vandi_b(tmp0, 0x1F); - tmpr = __lsx_vandi_b(tmp1, 0xF8); - reg1 = __lsx_vandi_b(tmp1, 0x07); - reg0 = __lsx_vsrli_b(tmp0, 5); - reg1 = __lsx_vslli_b(reg1, 3); - tmpg = __lsx_vor_v(reg1, reg0); - reg0 = __lsx_vslli_b(tmpb, 3); - reg1 = __lsx_vsrli_b(tmpb, 2); - tmpb = __lsx_vor_v(reg1, reg0); - reg0 = __lsx_vslli_b(tmpg, 2); - reg1 = __lsx_vsrli_b(tmpg, 4); - tmpg = __lsx_vor_v(reg1, reg0); - reg0 = __lsx_vsrli_b(tmpr, 5); - tmpr = __lsx_vor_v(tmpr, reg0); - reg0 = __lsx_vmaddwev_h_bu(const_1080, tmpb, const_25); - reg1 = __lsx_vmaddwod_h_bu(const_1080, tmpb, const_25); - reg0 = __lsx_vmaddwev_h_bu(reg0, tmpg, const_129); - reg1 = __lsx_vmaddwod_h_bu(reg1, tmpg, const_129); - reg0 = __lsx_vmaddwev_h_bu(reg0, tmpr, const_66); - reg1 = __lsx_vmaddwod_h_bu(reg1, tmpr, const_66); - dst0 = __lsx_vpackod_b(reg1, reg0); - __lsx_vst(dst0, dst_y, 0); - dst_y += 16; - src_rgb565 += 32; - } -} - -void RGB565ToUVRow_LSX(const uint8_t* src_rgb565, - int src_stride_rgb565, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 16; - const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565; - __m128i src0, src1, src2, src3; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m128i reg0, reg1, reg2, reg3, dst0; - __m128i const_112 = __lsx_vldi(0x470); - __m128i const_74 = __lsx_vldi(0x44A); - __m128i const_38 = __lsx_vldi(0x426); - __m128i const_94 = __lsx_vldi(0x45E); - __m128i const_18 = __lsx_vldi(0x412); - __m128i const_8000 = (__m128i)v2u64{0x8000800080008000, 0x8000800080008000}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_rgb565, 0, src_rgb565, 16, next_rgb565, 0, - next_rgb565, 16, src0, src1, src2, src3); - DUP2_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, tmp0, tmp2); - DUP2_ARG2(__lsx_vpickod_b, src1, src0, src3, src2, tmp1, tmp3); - tmpb = __lsx_vandi_b(tmp0, 0x1F); - tmpr = __lsx_vandi_b(tmp1, 0xF8); - nexb = __lsx_vandi_b(tmp2, 0x1F); - nexr = __lsx_vandi_b(tmp3, 0xF8); - reg1 = __lsx_vandi_b(tmp1, 0x07); - reg3 = __lsx_vandi_b(tmp3, 0x07); - reg0 = __lsx_vsrli_b(tmp0, 5); - reg1 = __lsx_vslli_b(reg1, 3); - reg2 = __lsx_vsrli_b(tmp2, 5); - reg3 = __lsx_vslli_b(reg3, 3); - tmpg = __lsx_vor_v(reg1, reg0); - nexg = __lsx_vor_v(reg2, reg3); - reg0 = __lsx_vslli_b(tmpb, 3); - reg1 = __lsx_vsrli_b(tmpb, 2); - reg2 = __lsx_vslli_b(nexb, 3); - reg3 = __lsx_vsrli_b(nexb, 2); - tmpb = __lsx_vor_v(reg1, reg0); - nexb = __lsx_vor_v(reg2, reg3); - reg0 = __lsx_vslli_b(tmpg, 2); - reg1 = __lsx_vsrli_b(tmpg, 4); - reg2 = __lsx_vslli_b(nexg, 2); - reg3 = __lsx_vsrli_b(nexg, 4); - tmpg = __lsx_vor_v(reg1, reg0); - nexg = __lsx_vor_v(reg2, reg3); - reg0 = __lsx_vsrli_b(tmpr, 5); - reg2 = __lsx_vsrli_b(nexr, 5); - tmpr = __lsx_vor_v(tmpr, reg0); - nexr = __lsx_vor_v(nexr, reg2); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, dst0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst0, dst_v, 0, 1); - dst_u += 8; - dst_v += 8; - src_rgb565 += 32; - next_rgb565 += 32; - } -} - -void RGB24ToUVRow_LSX(const uint8_t* src_rgb24, - int src_stride_rgb24, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - const uint8_t* next_rgb24 = src_rgb24 + src_stride_rgb24; - int len = width / 16; - __m128i src0, src1, src2; - __m128i nex0, nex1, nex2, dst0; - __m128i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m128i const_112 = __lsx_vldi(0x470); - __m128i const_74 = __lsx_vldi(0x44A); - __m128i const_38 = __lsx_vldi(0x426); - __m128i const_94 = __lsx_vldi(0x45E); - __m128i const_18 = __lsx_vldi(0x412); - __m128i const_8000 = (__m128i)v2u64{0x8000800080008000, 0x8000800080008000}; - __m128i shuff0_b = {0x15120F0C09060300, 0x00000000001E1B18}; - __m128i shuff1_b = {0x0706050403020100, 0x1D1A1714110A0908}; - __m128i shuff0_g = {0x1613100D0A070401, 0x00000000001F1C19}; - __m128i shuff1_g = {0x0706050403020100, 0x1E1B1815120A0908}; - __m128i shuff0_r = {0x1714110E0B080502, 0x0000000000001D1A}; - __m128i shuff1_r = {0x0706050403020100, 0x1F1C191613100908}; - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_rgb24, 0); - src1 = __lsx_vld(src_rgb24, 16); - src2 = __lsx_vld(src_rgb24, 32); - nex0 = __lsx_vld(next_rgb24, 0); - nex1 = __lsx_vld(next_rgb24, 16); - nex2 = __lsx_vld(next_rgb24, 32); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuff0_b, nex1, nex0, shuff0_b, tmpb, - nexb); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuff0_g, nex1, nex0, shuff0_g, tmpg, - nexg); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuff0_r, nex1, nex0, shuff0_r, tmpr, - nexr); - DUP2_ARG3(__lsx_vshuf_b, src2, tmpb, shuff1_b, nex2, nexb, shuff1_b, tmpb, - nexb); - DUP2_ARG3(__lsx_vshuf_b, src2, tmpg, shuff1_g, nex2, nexg, shuff1_g, tmpg, - nexg); - DUP2_ARG3(__lsx_vshuf_b, src2, tmpr, shuff1_r, nex2, nexr, shuff1_r, tmpr, - nexr); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, dst0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst0, dst_v, 0, 1); - dst_u += 8; - dst_v += 8; - src_rgb24 += 48; - next_rgb24 += 48; - } -} - -void RAWToUVRow_LSX(const uint8_t* src_raw, - int src_stride_raw, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - const uint8_t* next_raw = src_raw + src_stride_raw; - int len = width / 16; - __m128i src0, src1, src2; - __m128i nex0, nex1, nex2, dst0; - __m128i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m128i const_112 = __lsx_vldi(0x470); - __m128i const_74 = __lsx_vldi(0x44A); - __m128i const_38 = __lsx_vldi(0x426); - __m128i const_94 = __lsx_vldi(0x45E); - __m128i const_18 = __lsx_vldi(0x412); - __m128i const_8000 = (__m128i)v2u64{0x8000800080008000, 0x8000800080008000}; - __m128i shuff0_r = {0x15120F0C09060300, 0x00000000001E1B18}; - __m128i shuff1_r = {0x0706050403020100, 0x1D1A1714110A0908}; - __m128i shuff0_g = {0x1613100D0A070401, 0x00000000001F1C19}; - __m128i shuff1_g = {0x0706050403020100, 0x1E1B1815120A0908}; - __m128i shuff0_b = {0x1714110E0B080502, 0x0000000000001D1A}; - __m128i shuff1_b = {0x0706050403020100, 0x1F1C191613100908}; - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_raw, 0); - src1 = __lsx_vld(src_raw, 16); - src2 = __lsx_vld(src_raw, 32); - nex0 = __lsx_vld(next_raw, 0); - nex1 = __lsx_vld(next_raw, 16); - nex2 = __lsx_vld(next_raw, 32); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuff0_b, nex1, nex0, shuff0_b, tmpb, - nexb); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuff0_g, nex1, nex0, shuff0_g, tmpg, - nexg); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuff0_r, nex1, nex0, shuff0_r, tmpr, - nexr); - DUP2_ARG3(__lsx_vshuf_b, src2, tmpb, shuff1_b, nex2, nexb, shuff1_b, tmpb, - nexb); - DUP2_ARG3(__lsx_vshuf_b, src2, tmpg, shuff1_g, nex2, nexg, shuff1_g, tmpg, - nexg); - DUP2_ARG3(__lsx_vshuf_b, src2, tmpr, shuff1_r, nex2, nexr, shuff1_r, tmpr, - nexr); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, dst0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst0, dst_v, 0, 1); - dst_u += 8; - dst_v += 8; - src_raw += 48; - next_raw += 48; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr14, 31\n\t" + "vsrli.h $vr14, $vr14, 8\n\t" + "vldi $vr15, 128\n\t" + "vsrli.h $vr15, $vr15, 8\n\t" + "vslli.h $vr15, $vr15, 8\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.w $vr10, %[su], 0\n\t" + "vldrepl.w $vr11, %[sv], 0\n\t" + "vilvl.b $vr10, $vr10, $vr10\n\t" + "vilvl.b $vr11, $vr11, $vr11\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 3\n\t" + "vsrli.b $vr4, $vr4, 3\n\t" + "vilvl.b $vr2, $vr2, $vr2\n\t" + "vilvl.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr4, $vr4, $vr4\n\t" + "vand.v $vr2, $vr2, $vr14\n\t" + "vand.v $vr3, $vr3, $vr14\n\t" + "vand.v $vr4, $vr4, $vr14\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr4, $vr4, 10\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vor.v $vr2, $vr2, $vr15\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 4\n\t" + "addi.d %[sv], %[sv], 4\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr10", "$vr11", + "$vr14", "$vr15", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); } void NV12ToARGBRow_LSX(const uint8_t* src_y, @@ -1709,66 +829,47 @@ void NV12ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 8; - __m128i vec_y, vec_vu; - __m128i vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb; - __m128i vec_vrub, vec_vgug; - __m128i out_b, out_g, out_r; - __m128i const_80 = __lsx_vldi(0x480); - __m128i alpha = __lsx_vldi(0xFF); - __m128i zero = __lsx_vldi(0); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_vrub = __lsx_vilvl_h(vec_vr, vec_ub); - vec_vgug = __lsx_vilvl_h(vec_vg, vec_ug); - - for (x = 0; x < len; x++) { - vec_y = __lsx_vld(src_y, 0); - vec_vu = __lsx_vld(src_uv, 0); - YUVTORGB(vec_y, vec_vu, vec_vrub, vec_vgug, vec_yg, vec_yb, out_b, out_g, - out_r); - STOREARGB(alpha, out_r, out_g, out_b, dst_argb); - src_y += 8; - src_uv += 8; - } -} - -void NV12ToRGB565Row_LSX(const uint8_t* src_y, - const uint8_t* src_uv, - uint8_t* dst_rgb565, - const struct YuvConstants* yuvconstants, - int width) { - int x; - int len = width / 8; - __m128i vec_y, vec_vu; - __m128i vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb; - __m128i vec_vrub, vec_vgug; - __m128i out_b, out_g, out_r; - __m128i const_80 = __lsx_vldi(0x480); - __m128i zero = __lsx_vldi(0); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_vrub = __lsx_vilvl_h(vec_vr, vec_ub); - vec_vgug = __lsx_vilvl_h(vec_vg, vec_ug); - - for (x = 0; x < len; x++) { - vec_y = __lsx_vld(src_y, 0); - vec_vu = __lsx_vld(src_uv, 0); - YUVTORGB(vec_y, vec_vu, vec_vrub, vec_vgug, vec_yg, vec_yb, out_b, out_g, - out_r); - out_b = __lsx_vsrli_h(out_b, 3); - out_g = __lsx_vsrli_h(out_g, 2); - out_r = __lsx_vsrli_h(out_r, 3); - out_g = __lsx_vslli_h(out_g, 5); - out_r = __lsx_vslli_h(out_r, 11); - out_r = __lsx_vor_v(out_r, out_g); - out_r = __lsx_vor_v(out_r, out_b); - __lsx_vst(out_r, dst_rgb565, 0); - src_y += 8; - src_uv += 8; - dst_rgb565 += 16; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[suv], 0\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[suv], %[suv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); } void NV21ToARGBRow_LSX(const uint8_t* src_y, @@ -1776,257 +877,107 @@ void NV21ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 8; - __m128i vec_y, vec_uv; - __m128i vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb; - __m128i vec_ubvr, vec_ugvg; - __m128i out_b, out_g, out_r; - __m128i const_80 = __lsx_vldi(0x480); - __m128i alpha = __lsx_vldi(0xFF); - __m128i zero = __lsx_vldi(0); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ubvr = __lsx_vilvl_h(vec_ub, vec_vr); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - vec_y = __lsx_vld(src_y, 0); - vec_uv = __lsx_vld(src_vu, 0); - YUVTORGB(vec_y, vec_uv, vec_ubvr, vec_ugvg, vec_yg, vec_yb, out_r, out_g, - out_b); - STOREARGB(alpha, out_r, out_g, out_b, dst_argb); - src_y += 8; - src_vu += 8; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[suv], 0\n\t" + "vshuf4i.b $vr1, $vr1, 0xb1\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[suv], %[suv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_vu), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); } -void SobelRow_LSX(const uint8_t* src_sobelx, - const uint8_t* src_sobely, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 16; - __m128i src0, src1, tmp0; - __m128i out0, out1, out2, out3; - __m128i alpha = __lsx_vldi(0xFF); - __m128i shuff0 = {0x1001010110000000, 0x1003030310020202}; - __m128i shuff1 = __lsx_vaddi_bu(shuff0, 0x04); - __m128i shuff2 = __lsx_vaddi_bu(shuff1, 0x04); - __m128i shuff3 = __lsx_vaddi_bu(shuff2, 0x04); - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_sobelx, 0); - src1 = __lsx_vld(src_sobely, 0); - tmp0 = __lsx_vsadd_bu(src0, src1); - DUP4_ARG3(__lsx_vshuf_b, alpha, tmp0, shuff0, alpha, tmp0, shuff1, alpha, - tmp0, shuff2, alpha, tmp0, shuff3, out0, out1, out2, out3); - __lsx_vst(out0, dst_argb, 0); - __lsx_vst(out1, dst_argb, 16); - __lsx_vst(out2, dst_argb, 32); - __lsx_vst(out3, dst_argb, 48); - src_sobelx += 16; - src_sobely += 16; - dst_argb += 64; - } -} - -void SobelToPlaneRow_LSX(const uint8_t* src_sobelx, - const uint8_t* src_sobely, - uint8_t* dst_y, +void NV12ToRGB565Row_LSX(const uint8_t* src_y, + const uint8_t* src_uv, + uint8_t* dst_argb, + const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 32; - __m128i src0, src1, src2, src3, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_sobelx, 0, src_sobelx, 16, src0, src1); - DUP2_ARG2(__lsx_vld, src_sobely, 0, src_sobely, 16, src2, src3); - dst0 = __lsx_vsadd_bu(src0, src2); - dst1 = __lsx_vsadd_bu(src1, src3); - __lsx_vst(dst0, dst_y, 0); - __lsx_vst(dst1, dst_y, 16); - src_sobelx += 32; - src_sobely += 32; - dst_y += 32; - } -} - -void SobelXYRow_LSX(const uint8_t* src_sobelx, - const uint8_t* src_sobely, - uint8_t* dst_argb, - int width) { - int x; - int len = width / 16; - __m128i src_r, src_b, src_g; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i dst0, dst1, dst2, dst3; - __m128i alpha = __lsx_vldi(0xFF); - - for (x = 0; x < len; x++) { - src_r = __lsx_vld(src_sobelx, 0); - src_b = __lsx_vld(src_sobely, 0); - src_g = __lsx_vsadd_bu(src_r, src_b); - tmp0 = __lsx_vilvl_b(src_g, src_b); - tmp1 = __lsx_vilvh_b(src_g, src_b); - tmp2 = __lsx_vilvl_b(alpha, src_r); - tmp3 = __lsx_vilvh_b(alpha, src_r); - dst0 = __lsx_vilvl_h(tmp2, tmp0); - dst1 = __lsx_vilvh_h(tmp2, tmp0); - dst2 = __lsx_vilvl_h(tmp3, tmp1); - dst3 = __lsx_vilvh_h(tmp3, tmp1); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - src_sobelx += 16; - src_sobely += 16; - dst_argb += 64; - } -} - -void BGRAToUVRow_LSX(const uint8_t* src_bgra, - int src_stride_bgra, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - const uint8_t* next_bgra = src_bgra + src_stride_bgra; - int len = width / 16; - __m128i src0, src1, src2, src3; - __m128i nex0, nex1, nex2, nex3; - __m128i tmp0, tmp1, tmp2, tmp3, dst0; - __m128i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m128i const_112 = __lsx_vldi(0x470); - __m128i const_74 = __lsx_vldi(0x44A); - __m128i const_38 = __lsx_vldi(0x426); - __m128i const_94 = __lsx_vldi(0x45E); - __m128i const_18 = __lsx_vldi(0x412); - __m128i const_8000 = (__m128i)v2u64{0x8000800080008000, 0x8000800080008000}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_bgra, 0, src_bgra, 16, src_bgra, 32, src_bgra, 48, - src0, src1, src2, src3); - DUP4_ARG2(__lsx_vld, next_bgra, 0, next_bgra, 16, next_bgra, 32, next_bgra, - 48, nex0, nex1, nex2, nex3); - tmp0 = __lsx_vpickod_b(src1, src0); - tmp1 = __lsx_vpickev_b(src1, src0); - tmp2 = __lsx_vpickod_b(src3, src2); - tmp3 = __lsx_vpickev_b(src3, src2); - tmpb = __lsx_vpickod_b(tmp2, tmp0); - tmpr = __lsx_vpickev_b(tmp2, tmp0); - tmpg = __lsx_vpickod_b(tmp3, tmp1); - tmp0 = __lsx_vpickod_b(nex1, nex0); - tmp1 = __lsx_vpickev_b(nex1, nex0); - tmp2 = __lsx_vpickod_b(nex3, nex2); - tmp3 = __lsx_vpickev_b(nex3, nex2); - nexb = __lsx_vpickod_b(tmp2, tmp0); - nexr = __lsx_vpickev_b(tmp2, tmp0); - nexg = __lsx_vpickod_b(tmp3, tmp1); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, dst0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst0, dst_v, 0, 1); - dst_u += 8; - dst_v += 8; - src_bgra += 64; - next_bgra += 64; - } -} - -void ABGRToUVRow_LSX(const uint8_t* src_abgr, - int src_stride_abgr, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - const uint8_t* next_abgr = src_abgr + src_stride_abgr; - int len = width / 16; - __m128i src0, src1, src2, src3; - __m128i nex0, nex1, nex2, nex3; - __m128i tmp0, tmp1, tmp2, tmp3, dst0; - __m128i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m128i const_112 = __lsx_vldi(0x470); - __m128i const_74 = __lsx_vldi(0x44A); - __m128i const_38 = __lsx_vldi(0x426); - __m128i const_94 = __lsx_vldi(0x45E); - __m128i const_18 = __lsx_vldi(0x412); - __m128i const_8000 = (__m128i)v2u64{0x8000800080008000, 0x8000800080008000}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_abgr, 0, src_abgr, 16, src_abgr, 32, src_abgr, 48, - src0, src1, src2, src3); - DUP4_ARG2(__lsx_vld, next_abgr, 0, next_abgr, 16, next_abgr, 32, next_abgr, - 48, nex0, nex1, nex2, nex3); - tmp0 = __lsx_vpickev_b(src1, src0); - tmp1 = __lsx_vpickod_b(src1, src0); - tmp2 = __lsx_vpickev_b(src3, src2); - tmp3 = __lsx_vpickod_b(src3, src2); - tmpb = __lsx_vpickod_b(tmp2, tmp0); - tmpr = __lsx_vpickev_b(tmp2, tmp0); - tmpg = __lsx_vpickev_b(tmp3, tmp1); - tmp0 = __lsx_vpickev_b(nex1, nex0); - tmp1 = __lsx_vpickod_b(nex1, nex0); - tmp2 = __lsx_vpickev_b(nex3, nex2); - tmp3 = __lsx_vpickod_b(nex3, nex2); - nexb = __lsx_vpickod_b(tmp2, tmp0); - nexr = __lsx_vpickev_b(tmp2, tmp0); - nexg = __lsx_vpickev_b(tmp3, tmp1); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, dst0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst0, dst_v, 0, 1); - dst_u += 8; - dst_v += 8; - src_abgr += 64; - next_abgr += 64; - } -} - -void RGBAToUVRow_LSX(const uint8_t* src_rgba, - int src_stride_rgba, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - const uint8_t* next_rgba = src_rgba + src_stride_rgba; - int len = width / 16; - __m128i src0, src1, src2, src3; - __m128i nex0, nex1, nex2, nex3; - __m128i tmp0, tmp1, tmp2, tmp3, dst0; - __m128i tmpb, tmpg, tmpr, nexb, nexg, nexr; - __m128i const_112 = __lsx_vldi(0x470); - __m128i const_74 = __lsx_vldi(0x44A); - __m128i const_38 = __lsx_vldi(0x426); - __m128i const_94 = __lsx_vldi(0x45E); - __m128i const_18 = __lsx_vldi(0x412); - __m128i const_8000 = (__m128i)v2u64{0x8000800080008000, 0x8000800080008000}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_rgba, 0, src_rgba, 16, src_rgba, 32, src_rgba, 48, - src0, src1, src2, src3); - DUP4_ARG2(__lsx_vld, next_rgba, 0, next_rgba, 16, next_rgba, 32, next_rgba, - 48, nex0, nex1, nex2, nex3); - tmp0 = __lsx_vpickod_b(src1, src0); - tmp1 = __lsx_vpickev_b(src1, src0); - tmp2 = __lsx_vpickod_b(src3, src2); - tmp3 = __lsx_vpickev_b(src3, src2); - tmpr = __lsx_vpickod_b(tmp2, tmp0); - tmpb = __lsx_vpickev_b(tmp2, tmp0); - tmpg = __lsx_vpickod_b(tmp3, tmp1); - tmp0 = __lsx_vpickod_b(nex1, nex0); - tmp1 = __lsx_vpickev_b(nex1, nex0); - tmp2 = __lsx_vpickod_b(nex3, nex2); - tmp3 = __lsx_vpickev_b(nex3, nex2); - nexr = __lsx_vpickod_b(tmp2, tmp0); - nexb = __lsx_vpickev_b(tmp2, tmp0); - nexg = __lsx_vpickod_b(tmp3, tmp1); - RGBTOUV(tmpb, tmpg, tmpr, nexb, nexg, nexr, dst0); - __lsx_vstelm_d(dst0, dst_u, 0, 0); - __lsx_vstelm_d(dst0, dst_v, 0, 1); - dst_u += 8; - dst_v += 8; - src_rgba += 64; - next_rgba += 64; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr14, 31\n\t" + "vsrli.h $vr14, $vr14, 8\n\t" + "vldi $vr15, 63\n\t" + "vsrli.h $vr15, $vr15, 8\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.d $vr1, %[suv], 0\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 2\n\t" + "vsrli.b $vr4, $vr4, 3\n\t" + "vilvl.b $vr2, $vr2, $vr2\n\t" + "vilvl.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr4, $vr4, $vr4\n\t" + "vand.v $vr2, $vr2, $vr14\n\t" + "vand.v $vr3, $vr3, $vr15\n\t" + "vand.v $vr4, $vr4, $vr14\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr4, $vr4, 11\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vor.v $vr2, $vr2, $vr4\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[suv], %[suv], 8\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [suv] "+r"(src_uv), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr14", "$vr15", + "$vr20", "$vr21", "$vr22", "$vr23", "$vr24"); } void I444ToARGBRow_LSX(const uint8_t* src_y, @@ -2035,362 +986,865 @@ void I444ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m128i vec_y, vec_u, vec_v, out_b, out_g, out_r; - __m128i vec_yl, vec_yh, vec_ul, vec_vl, vec_uh, vec_vh; - __m128i vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb, vec_ugvg; - __m128i const_80 = __lsx_vldi(0x480); - __m128i alpha = __lsx_vldi(0xFF); - __m128i zero = __lsx_vldi(0); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_ugvg = __lsx_vilvl_h(vec_ug, vec_vg); - - for (x = 0; x < len; x++) { - vec_y = __lsx_vld(src_y, 0); - vec_u = __lsx_vld(src_u, 0); - vec_v = __lsx_vld(src_v, 0); - vec_yl = __lsx_vilvl_b(vec_y, vec_y); - vec_ul = __lsx_vilvl_b(zero, vec_u); - vec_vl = __lsx_vilvl_b(zero, vec_v); - I444TORGB(vec_yl, vec_ul, vec_vl, vec_ub, vec_vr, vec_ugvg, vec_yg, vec_yb, - out_b, out_g, out_r); - STOREARGB(alpha, out_r, out_g, out_b, dst_argb); - vec_yh = __lsx_vilvh_b(vec_y, vec_y); - vec_uh = __lsx_vilvh_b(zero, vec_u); - vec_vh = __lsx_vilvh_b(zero, vec_v); - I444TORGB(vec_yh, vec_uh, vec_vh, vec_ub, vec_vr, vec_ugvg, vec_yg, vec_yb, - out_b, out_g, out_r); - STOREARGB(alpha, out_r, out_g, out_b, dst_argb); - src_y += 16; - src_u += 16; - src_v += 16; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vldrepl.d $vr10, %[su], 0\n\t" + "vldrepl.d $vr11, %[sv], 0\n\t" + "vilvl.b $vr1, $vr11, $vr10\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[su], %[su], 8\n\t" + "addi.d %[sv], %[sv], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [su] "+r"(src_u), [sv] "+r"(src_v), + [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr20", "$vr21", "$vr22", + "$vr23", "$vr24"); } void I400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 16; - __m128i vec_y, vec_yl, vec_yh, out0; - __m128i y_ev, y_od, dst0, dst1, dst2, dst3; - __m128i temp0, temp1; - __m128i alpha = __lsx_vldi(0xFF); - __m128i vec_yg = __lsx_vreplgr2vr_h(yuvconstants->kYToRgb[0]); - __m128i vec_yb = __lsx_vreplgr2vr_w(yuvconstants->kYBiasToRgb[0]); - - for (x = 0; x < len; x++) { - vec_y = __lsx_vld(src_y, 0); - vec_yl = __lsx_vilvl_b(vec_y, vec_y); - y_ev = __lsx_vmulwev_w_hu_h(vec_yl, vec_yg); - y_od = __lsx_vmulwod_w_hu_h(vec_yl, vec_yg); - y_ev = __lsx_vsrai_w(y_ev, 16); - y_od = __lsx_vsrai_w(y_od, 16); - y_ev = __lsx_vadd_w(y_ev, vec_yb); - y_od = __lsx_vadd_w(y_od, vec_yb); - y_ev = __lsx_vsrai_w(y_ev, 6); - y_od = __lsx_vsrai_w(y_od, 6); - y_ev = __lsx_vclip255_w(y_ev); - y_od = __lsx_vclip255_w(y_od); - out0 = __lsx_vpackev_h(y_od, y_ev); - temp0 = __lsx_vpackev_b(out0, out0); - temp1 = __lsx_vpackev_b(alpha, out0); - dst0 = __lsx_vilvl_h(temp1, temp0); - dst1 = __lsx_vilvh_h(temp1, temp0); - vec_yh = __lsx_vilvh_b(vec_y, vec_y); - y_ev = __lsx_vmulwev_w_hu_h(vec_yh, vec_yg); - y_od = __lsx_vmulwod_w_hu_h(vec_yh, vec_yg); - y_ev = __lsx_vsrai_w(y_ev, 16); - y_od = __lsx_vsrai_w(y_od, 16); - y_ev = __lsx_vadd_w(y_ev, vec_yb); - y_od = __lsx_vadd_w(y_od, vec_yb); - y_ev = __lsx_vsrai_w(y_ev, 6); - y_od = __lsx_vsrai_w(y_od, 6); - y_ev = __lsx_vclip255_w(y_ev); - y_od = __lsx_vclip255_w(y_od); - out0 = __lsx_vpackev_h(y_od, y_ev); - temp0 = __lsx_vpackev_b(out0, out0); - temp1 = __lsx_vpackev_b(alpha, out0); - dst2 = __lsx_vilvl_h(temp1, temp0); - dst3 = __lsx_vilvh_h(temp1, temp0); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - dst_argb += 64; - src_y += 16; - } + asm volatile( + "vld $vr23, %[c], 96\n\t" + "vld $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sy], 0\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vssrani.bu.h $vr2, $vr0, 6\n\t" + "vilvl.b $vr6, $vr2, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr2\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr2", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9", + "$vr23", "$vr24"); } void J400ToARGBRow_LSX(const uint8_t* src_y, uint8_t* dst_argb, int width) { - int x; - int len = width / 16; - __m128i vec_y, dst0, dst1, dst2, dst3; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i alpha = __lsx_vldi(0xFF); - - for (x = 0; x < len; x++) { - vec_y = __lsx_vld(src_y, 0); - tmp0 = __lsx_vilvl_b(vec_y, vec_y); - tmp1 = __lsx_vilvh_b(vec_y, vec_y); - tmp2 = __lsx_vilvl_b(alpha, vec_y); - tmp3 = __lsx_vilvh_b(alpha, vec_y); - dst0 = __lsx_vilvl_h(tmp2, tmp0); - dst1 = __lsx_vilvh_h(tmp2, tmp0); - dst2 = __lsx_vilvl_h(tmp3, tmp1); - dst3 = __lsx_vilvh_h(tmp3, tmp1); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - dst_argb += 64; - src_y += 16; - } + asm volatile( + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr2, %[sy], 0\n\t" + "vilvl.b $vr6, $vr2, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr2\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sy] "+r"(src_y), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr2", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9"); } void YUY2ToARGBRow_LSX(const uint8_t* src_yuy2, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 8; - __m128i src0, vec_y, vec_vu; - __m128i vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb; - __m128i vec_vrub, vec_vgug; - __m128i out_b, out_g, out_r; - __m128i const_80 = __lsx_vldi(0x480); - __m128i zero = __lsx_vldi(0); - __m128i alpha = __lsx_vldi(0xFF); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_vrub = __lsx_vilvl_h(vec_vr, vec_ub); - vec_vgug = __lsx_vilvl_h(vec_vg, vec_ug); - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_yuy2, 0); - vec_y = __lsx_vpickev_b(src0, src0); - vec_vu = __lsx_vpickod_b(src0, src0); - YUVTORGB(vec_y, vec_vu, vec_vrub, vec_vgug, vec_yg, vec_yb, out_b, out_g, - out_r); - STOREARGB(alpha, out_r, out_g, out_b, dst_argb); - src_yuy2 += 16; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vld $vr10, %[src], 0\n\t" + "vpickev.b $vr0, $vr10, $vr10\n\t" + "vpickod.b $vr1, $vr10, $vr10\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_yuy2), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr20", "$vr21", "$vr22", "$vr23", + "$vr24"); } void UYVYToARGBRow_LSX(const uint8_t* src_uyvy, uint8_t* dst_argb, const struct YuvConstants* yuvconstants, int width) { - int x; - int len = width / 8; - __m128i src0, vec_y, vec_vu; - __m128i vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb; - __m128i vec_vrub, vec_vgug; - __m128i out_b, out_g, out_r; - __m128i const_80 = __lsx_vldi(0x480); - __m128i zero = __lsx_vldi(0); - __m128i alpha = __lsx_vldi(0xFF); - - YUVTORGB_SETUP(yuvconstants, vec_vr, vec_ub, vec_vg, vec_ug, vec_yg, vec_yb); - vec_vrub = __lsx_vilvl_h(vec_vr, vec_ub); - vec_vgug = __lsx_vilvl_h(vec_vg, vec_ug); - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_uyvy, 0); - vec_y = __lsx_vpickod_b(src0, src0); - vec_vu = __lsx_vpickev_b(src0, src0); - YUVTORGB(vec_y, vec_vu, vec_vrub, vec_vgug, vec_yg, vec_yb, out_b, out_g, - out_r); - STOREARGB(alpha, out_r, out_g, out_b, dst_argb); - src_uyvy += 16; - } + asm volatile( + "vldrepl.w $vr20, %[c], 0\n\t" + "vldrepl.w $vr21, %[c], 32\n\t" + "vldrepl.w $vr22, %[c], 64\n\t" + "vldrepl.h $vr23, %[c], 96\n\t" + "vldrepl.h $vr24, %[c], 128\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vld $vr10, %[src], 0\n\t" + "vpickod.b $vr0, $vr10, $vr10\n\t" + "vpickev.b $vr1, $vr10, $vr10\n\t" + "vilvl.h $vr1, $vr1, $vr1\n\t" + "vilvl.b $vr0, $vr0, $vr0\n\t" + "vxori.b $vr1, $vr1, 0x80\n\t" + "vmuh.hu $vr0, $vr0, $vr23\n\t" + "vadd.h $vr0, $vr0, $vr24\n\t" + "vmulwev.h.bu.b $vr2, $vr20, $vr1\n\t" + "vmulwod.h.bu.b $vr4, $vr22, $vr1\n\t" + "vmulwev.h.bu.b $vr3, $vr21, $vr1\n\t" + "vmaddwod.h.bu.b $vr3, $vr21, $vr1\n\t" + "vsadd.h $vr2, $vr0, $vr2\n\t" + "vsadd.h $vr4, $vr0, $vr4\n\t" + "vssub.h $vr3, $vr0, $vr3\n\t" + "vssrani.bu.h $vr2, $vr2, 6\n\t" + "vssrani.bu.h $vr3, $vr3, 6\n\t" + "vssrani.bu.h $vr4, $vr4, 6\n\t" + "vilvl.b $vr6, $vr3, $vr2\n\t" + "vilvl.b $vr7, $vr5, $vr4\n\t" + "vilvl.h $vr8, $vr7, $vr6\n\t" + "vilvh.h $vr9, $vr7, $vr6\n\t" + "vst $vr8, %[dst], 0\n\t" + "vst $vr9, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_uyvy), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [c] "r"(yuvconstants) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr20", "$vr21", "$vr22", "$vr23", + "$vr24"); } -void InterpolateRow_LSX(uint8_t* dst_ptr, - const uint8_t* src_ptr, - ptrdiff_t src_stride, - int width, - int32_t source_y_fraction) { - int x; - int y1_fraction = source_y_fraction; - int y0_fraction = 256 - y1_fraction; - const uint8_t* nex_ptr = src_ptr + src_stride; - uint16_t y_fractions; - int len = width / 32; - __m128i src0, src1, nex0, nex1; - __m128i dst0, dst1, y_frac; - __m128i tmp0, tmp1, tmp2, tmp3; - __m128i const_128 = __lsx_vldi(0x480); - - if (y1_fraction == 0) { - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); - __lsx_vst(src0, dst_ptr, 0); - __lsx_vst(src1, dst_ptr, 16); - src_ptr += 32; - dst_ptr += 32; - } - return; - } - - if (y1_fraction == 128) { - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); - DUP2_ARG2(__lsx_vld, nex_ptr, 0, nex_ptr, 16, nex0, nex1); - dst0 = __lsx_vavgr_bu(src0, nex0); - dst1 = __lsx_vavgr_bu(src1, nex1); - __lsx_vst(dst0, dst_ptr, 0); - __lsx_vst(dst1, dst_ptr, 16); - src_ptr += 32; - nex_ptr += 32; - dst_ptr += 32; - } - return; - } - - y_fractions = (uint16_t)(y0_fraction + (y1_fraction << 8)); - y_frac = __lsx_vreplgr2vr_h(y_fractions); - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); - DUP2_ARG2(__lsx_vld, nex_ptr, 0, nex_ptr, 16, nex0, nex1); - tmp0 = __lsx_vilvl_b(nex0, src0); - tmp1 = __lsx_vilvh_b(nex0, src0); - tmp2 = __lsx_vilvl_b(nex1, src1); - tmp3 = __lsx_vilvh_b(nex1, src1); - tmp0 = __lsx_vdp2add_h_bu(const_128, tmp0, y_frac); - tmp1 = __lsx_vdp2add_h_bu(const_128, tmp1, y_frac); - tmp2 = __lsx_vdp2add_h_bu(const_128, tmp2, y_frac); - tmp3 = __lsx_vdp2add_h_bu(const_128, tmp3, y_frac); - dst0 = __lsx_vsrlni_b_h(tmp1, tmp0, 8); - dst1 = __lsx_vsrlni_b_h(tmp3, tmp2, 8); - __lsx_vst(dst0, dst_ptr, 0); - __lsx_vst(dst1, dst_ptr, 16); - src_ptr += 32; - nex_ptr += 32; - dst_ptr += 32; - } +static LIBYUV_NOINLINE void ARGBTo24Helper_LSX(const uint8_t* src_argb, + uint8_t* dst_24, + const uint8_t* shuf, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vld $vr2, %[src], 32\n\t" + "vld $vr3, %[src], 48\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" + "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" + "vshuf.b $vr2, $vr2, $vr2, $vr4\n\t" + "vshuf.b $vr3, $vr3, $vr3, $vr4\n\t" + "vstelm.d $vr0, %[dst], 0, 0\n\t" + "vstelm.w $vr0, %[dst], 8, 2\n\t" + "vstelm.w $vr1, %[dst], 12, 0\n\t" + "vstelm.w $vr1, %[dst], 16, 1\n\t" + "vstelm.w $vr1, %[dst], 20, 2\n\t" + "vstelm.d $vr2, %[dst], 24, 0\n\t" + "vstelm.w $vr2, %[dst], 32, 2\n\t" + "vstelm.w $vr3, %[dst], 36, 0\n\t" + "vstelm.w $vr3, %[dst], 40, 1\n\t" + "vstelm.w $vr3, %[dst], 44, 2\n\t" + "addi.d %[src], %[src], 64\n\t" + "addi.d %[dst], %[dst], 48\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_24), [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); } -void ARGBSetRow_LSX(uint8_t* dst_argb, uint32_t v32, int width) { - int x; - int len = width / 4; - __m128i dst0 = __lsx_vreplgr2vr_w(v32); +void ARGBToRGB24Row_LSX(const uint8_t* src_argb, + uint8_t* dst_rgb24, + int width) { + static const uint8_t kShuf[16] = {0, 1, 2, 4, 5, 6, 8, 9, + 10, 12, 13, 14, 0, 0, 0, 0}; + ARGBTo24Helper_LSX(src_argb, dst_rgb24, kShuf, width); +} - for (x = 0; x < len; x++) { - __lsx_vst(dst0, dst_argb, 0); - dst_argb += 16; - } +void ARGBToRAWRow_LSX(const uint8_t* src_argb, uint8_t* dst_raw, int width) { + static const uint8_t kShuf[16] = {2, 1, 0, 6, 5, 4, 10, 9, + 8, 14, 13, 12, 0, 0, 0, 0}; + ARGBTo24Helper_LSX(src_argb, dst_raw, kShuf, width); +} + +static LIBYUV_NOINLINE void RGB24ToARGBHelper_LSX(const uint8_t* src_24, + uint8_t* dst_argb, + const uint8_t* shuf, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "vld $vr5, %[shuf], 16\n\t" + "vldi $vr6, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 12\n\t" + "vld $vr2, %[src], 24\n\t" + "vld $vr3, %[src], 32\n\t" + "vshuf.b $vr0, $vr6, $vr0, $vr4\n\t" + "vshuf.b $vr1, $vr6, $vr1, $vr4\n\t" + "vshuf.b $vr2, $vr6, $vr2, $vr4\n\t" + "vshuf.b $vr3, $vr6, $vr3, $vr5\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "vst $vr2, %[dst], 32\n\t" + "vst $vr3, %[dst], 48\n\t" + "addi.d %[src], %[src], 48\n\t" + "addi.d %[dst], %[dst], 64\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_24), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} + +void RGB24ToARGBRow_LSX(const uint8_t* src_rgb24, + uint8_t* dst_argb, + int width) { + static const uint8_t kShuf[32] = {0, 1, 2, 16, 3, 4, 5, 16, 6, 7, 8, + 16, 9, 10, 11, 16, 4, 5, 6, 16, 7, 8, + 9, 16, 10, 11, 12, 16, 13, 14, 15, 16}; + RGB24ToARGBHelper_LSX(src_rgb24, dst_argb, kShuf, width); +} + +void RAWToARGBRow_LSX(const uint8_t* src_raw, uint8_t* dst_argb, int width) { + static const uint8_t kShuf[32] = {2, 1, 0, 16, 5, 4, 3, 16, 8, 7, 6, + 16, 11, 10, 9, 16, 6, 5, 4, 16, 9, 8, + 7, 16, 12, 11, 10, 16, 15, 14, 13, 16}; + RGB24ToARGBHelper_LSX(src_raw, dst_argb, kShuf, width); } void RAWToRGB24Row_LSX(const uint8_t* src_raw, uint8_t* dst_rgb24, int width) { - int x; - int len = width / 16; - __m128i src0, src1, src2; - __m128i dst0, dst1, dst2; - __m128i shuf0 = {0x0708030405000102, 0x110C0D0E090A0B06}; - __m128i shuf1 = {0x1516171213140F10, 0x1F1E1B1C1D18191A}; - __m128i shuf2 = {0x090405060102031E, 0x0D0E0F0A0B0C0708}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_raw, 0, src_raw, 16, src0, src1); - src2 = __lsx_vld(src_raw, 32); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuf0, src1, src0, shuf1, dst0, dst1); - dst2 = __lsx_vshuf_b(src1, src2, shuf2); - dst1 = __lsx_vextrins_b(dst1, src2, 0xE0); - __lsx_vst(dst0, dst_rgb24, 0); - __lsx_vst(dst1, dst_rgb24, 16); - __lsx_vst(dst2, dst_rgb24, 32); - dst_rgb24 += 48; - src_raw += 48; - } + static const uint8_t kShuf[32] = {2, 1, 0, 5, 4, 3, 8, 7, 6, 11, 10, + 9, 0, 0, 0, 0, 6, 5, 4, 9, 8, 7, + 12, 11, 10, 15, 14, 13, 0, 0, 0, 0}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "vld $vr5, %[shuf], 16\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 12\n\t" + "vld $vr2, %[src], 24\n\t" + "vld $vr3, %[src], 32\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" + "vshuf.b $vr1, $vr1, $vr1, $vr4\n\t" + "vshuf.b $vr2, $vr2, $vr2, $vr4\n\t" + "vshuf.b $vr3, $vr3, $vr3, $vr5\n\t" + "vstelm.d $vr0, %[dst], 0, 0\n\t" + "vstelm.w $vr0, %[dst], 8, 2\n\t" + "vstelm.w $vr1, %[dst], 12, 0\n\t" + "vstelm.w $vr1, %[dst], 16, 1\n\t" + "vstelm.w $vr1, %[dst], 20, 2\n\t" + "vstelm.d $vr2, %[dst], 24, 0\n\t" + "vstelm.w $vr2, %[dst], 32, 2\n\t" + "vstelm.w $vr3, %[dst], 36, 0\n\t" + "vstelm.w $vr3, %[dst], 40, 1\n\t" + "vstelm.w $vr3, %[dst], 44, 2\n\t" + "addi.d %[src], %[src], 48\n\t" + "addi.d %[dst], %[dst], 48\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_raw), [dst] "+r"(dst_rgb24), [cnt] "+r"(width) + : [shuf] "r"(kShuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); } -void MergeUVRow_LSX(const uint8_t* src_u, - const uint8_t* src_v, - uint8_t* dst_uv, +void ARGBToRGB565Row_LSX(const uint8_t* src_argb, uint8_t* dst_rgb, int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 2\n\t" + "vand.v $vr4, $vr2, $vr6\n\t" + "vsrli.h $vr2, $vr2, 8\n\t" + "vand.v $vr3, $vr3, $vr7\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr2, $vr2, 11\n\t" + "vor.v $vr4, $vr4, $vr3\n\t" + "vor.v $vr4, $vr4, $vr2\n\t" + "vst $vr4, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7"); +} + +void ARGBToRGB565DitherRow_LSX(const uint8_t* src_argb, + uint8_t* dst_rgb, + uint32_t dither4, + int width) { + asm volatile( + "vreplgr2vr.w $vr5, %[dith]\n\t" + "vilvl.b $vr5, $vr5, $vr5\n\t" + "vilvl.h $vr5, $vr5, $vr5\n\t" + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vsadd.bu $vr0, $vr0, $vr5\n\t" + "vpickev.b $vr2, $vr0, $vr0\n\t" + "vpickod.b $vr3, $vr0, $vr0\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vsrli.b $vr3, $vr3, 2\n\t" + "vand.v $vr4, $vr2, $vr6\n\t" + "vsrli.h $vr2, $vr2, 8\n\t" + "vand.v $vr3, $vr3, $vr7\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr2, $vr2, 11\n\t" + "vor.v $vr4, $vr4, $vr3\n\t" + "vor.v $vr4, $vr4, $vr2\n\t" + "vstelm.d $vr4, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : [dith] "r"(dither4) + : "memory", "$vr0", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", "$vr7"); +} + +void ARGBToARGB1555Row_LSX(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vsrli.b $vr2, $vr2, 3\n\t" + "vand.v $vr4, $vr2, $vr6\n\t" + "vsrli.h $vr2, $vr2, 8\n\t" + "vsrli.h $vr5, $vr3, 15\n\t" + "vsrli.b $vr3, $vr3, 3\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vslli.h $vr3, $vr3, 5\n\t" + "vslli.h $vr2, $vr2, 10\n\t" + "vslli.h $vr5, $vr5, 15\n\t" + "vor.v $vr4, $vr4, $vr3\n\t" + "vor.v $vr4, $vr4, $vr2\n\t" + "vor.v $vr4, $vr4, $vr5\n\t" + "vst $vr4, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} + +void ARGBToARGB4444Row_LSX(const uint8_t* src_argb, + uint8_t* dst_rgb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vsrli.b $vr2, $vr2, 4\n\t" + "vsrli.b $vr3, $vr3, 4\n\t" + "vslli.b $vr3, $vr3, 4\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_rgb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} + +void RGB565ToARGBRow_LSX(const uint8_t* src_rgb565, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "vldi $vr5, 0xff\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr1, $vr1, $vr1\n\t" + "vpickev.b $vr2, $vr2, $vr2\n\t" + "vpickev.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr8, $vr2, $vr1\n\t" + "vilvl.b $vr9, $vr5, $vr3\n\t" + "vilvl.h $vr10, $vr9, $vr8\n\t" + "vilvh.h $vr11, $vr9, $vr8\n\t" + "vst $vr10, %[dst], 0\n\t" + "vst $vr11, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgb565), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11"); +} + +void ARGB1555ToARGBRow_LSX(const uint8_t* src_argb1555, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vsrai.h $vr5, $vr0, 15\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr1, $vr1, $vr1\n\t" + "vpickev.b $vr2, $vr2, $vr2\n\t" + "vpickev.b $vr3, $vr3, $vr3\n\t" + "vpickev.b $vr5, $vr5, $vr5\n\t" + "vilvl.b $vr8, $vr2, $vr1\n\t" + "vilvl.b $vr9, $vr5, $vr3\n\t" + "vilvl.h $vr10, $vr9, $vr8\n\t" + "vilvh.h $vr11, $vr9, $vr8\n\t" + "vst $vr10, %[dst], 0\n\t" + "vst $vr11, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb1555), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr8", "$vr9", "$vr10", "$vr11"); +} + +void ARGB4444ToARGBRow_LSX(const uint8_t* src_argb4444, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr6, 0x0f\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.b $vr2, $vr0, 4\n\t" + "vslli.b $vr3, $vr1, 4\n\t" + "vor.v $vr1, $vr1, $vr3\n\t" + "vslli.b $vr3, $vr2, 4\n\t" + "vor.v $vr2, $vr2, $vr3\n\t" + "vilvl.b $vr4, $vr2, $vr1\n\t" + "vilvh.b $vr5, $vr2, $vr1\n\t" + "vst $vr4, %[dst], 0\n\t" + "vst $vr5, %[dst], 16\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb4444), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} + +void ARGBMultiplyRow_LSX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vmulwev.h.bu $vr2, $vr0, $vr1\n\t" + "vmulwod.h.bu $vr3, $vr0, $vr1\n\t" + "vsrari.h $vr2, $vr2, 8\n\t" + "vsrari.h $vr3, $vr3, 8\n\t" + "vpackev.b $vr2, $vr3, $vr2\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); +} + +void ARGBAddRow_LSX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, int width) { - int x; - int len = width / 16; - __m128i src0, src1, dst0, dst1; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_u, 0, src_v, 0, src0, src1); - dst0 = __lsx_vilvl_b(src1, src0); - dst1 = __lsx_vilvh_b(src1, src0); - __lsx_vst(dst0, dst_uv, 0); - __lsx_vst(dst1, dst_uv, 16); - src_u += 16; - src_v += 16; - dst_uv += 32; - } + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vsadd.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); } -void ARGBExtractAlphaRow_LSX(const uint8_t* src_argb, - uint8_t* dst_a, - int width) { - int x; - int len = width / 16; - __m128i src0, src1, src2, src3, tmp0, tmp1, dst0; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src_argb, 32, src_argb, 48, - src0, src1, src2, src3); - tmp0 = __lsx_vpickod_b(src1, src0); - tmp1 = __lsx_vpickod_b(src3, src2); - dst0 = __lsx_vpickod_b(tmp1, tmp0); - __lsx_vst(dst0, dst_a, 0); - src_argb += 64; - dst_a += 16; - } +void ARGBSubtractRow_LSX(const uint8_t* src_argb, + const uint8_t* src_argb1, + uint8_t* dst_argb, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vssub.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); } -void ARGBBlendRow_LSX(const uint8_t* src_argb, +void ARGBAttenuateRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + int width) { + static const uint8_t kAlpha[16] = {3, 3, 3, 16, 7, 7, 7, 16, + 11, 11, 11, 16, 15, 15, 15, 16}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "vldi $vr5, 0xff\n\t" + "vldi $vr6, 255\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf.b $vr1, $vr5, $vr0, $vr4\n\t" + "vmulwev.h.bu $vr2, $vr0, $vr1\n\t" + "vmulwod.h.bu $vr3, $vr0, $vr1\n\t" + "vadd.h $vr2, $vr2, $vr6\n\t" + "vadd.h $vr3, $vr3, $vr6\n\t" + "vpackod.b $vr2, $vr3, $vr2\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(kAlpha) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6"); +} + +void ARGBBlendRow_LSX(const uint8_t* src_argb0, const uint8_t* src_argb1, uint8_t* dst_argb, int width) { - int x; - int len = width / 8; - __m128i src0, src1, src2, src3; - __m128i tmp0, tmp1, dst0, dst1; - __m128i reg0, reg1, reg2, reg3; - __m128i a0, a1, a2, a3; - __m128i const_256 = __lsx_vldi(0x500); - __m128i zero = __lsx_vldi(0); - __m128i alpha = __lsx_vldi(0xFF); - __m128i control = (__m128i)v2u64{0xFF000000FF000000, 0xFF000000FF000000}; + static const uint8_t kAlpha[16] = {3, 3, 3, 3, 7, 7, 7, 7, + 11, 11, 11, 11, 15, 15, 15, 15}; + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "vldi $vr5, 0xff\n\t" + "vslli.w $vr5, $vr5, 24\n\t" + "vldi $vr6, 1\n\t" + "vslli.h $vr6, $vr6, 8\n\t" + "vldi $vr8, 0\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vshuf.b $vr2, $vr0, $vr0, $vr4\n\t" + "vilvl.b $vr3, $vr8, $vr2\n\t" + "vilvh.b $vr2, $vr8, $vr2\n\t" + "vsub.h $vr3, $vr6, $vr3\n\t" + "vsub.h $vr2, $vr6, $vr2\n\t" + "vilvl.b $vr7, $vr8, $vr1\n\t" + "vilvh.b $vr1, $vr8, $vr1\n\t" + "vmul.h $vr7, $vr7, $vr3\n\t" + "vmul.h $vr1, $vr1, $vr2\n\t" + "vsrlni.b.h $vr1, $vr7, 8\n\t" + "vsadd.bu $vr0, $vr0, $vr1\n\t" + "vor.v $vr0, $vr0, $vr5\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb0), [s1] "+r"(src_argb1), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : [shuf] "r"(kAlpha) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8"); +} - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src_argb1, 0, src_argb1, 16, - src0, src1, src2, src3); - tmp0 = __lsx_vshuf4i_b(src0, 0xFF); - tmp1 = __lsx_vshuf4i_b(src1, 0xFF); - a0 = __lsx_vilvl_b(zero, tmp0); - a1 = __lsx_vilvh_b(zero, tmp0); - a2 = __lsx_vilvl_b(zero, tmp1); - a3 = __lsx_vilvh_b(zero, tmp1); - reg0 = __lsx_vilvl_b(zero, src2); - reg1 = __lsx_vilvh_b(zero, src2); - reg2 = __lsx_vilvl_b(zero, src3); - reg3 = __lsx_vilvh_b(zero, src3); - DUP4_ARG2(__lsx_vsub_h, const_256, a0, const_256, a1, const_256, a2, - const_256, a3, a0, a1, a2, a3); - DUP4_ARG2(__lsx_vmul_h, a0, reg0, a1, reg1, a2, reg2, a3, reg3, reg0, reg1, - reg2, reg3); - DUP2_ARG3(__lsx_vsrani_b_h, reg1, reg0, 8, reg3, reg2, 8, dst0, dst1); - dst0 = __lsx_vsadd_bu(dst0, src0); - dst1 = __lsx_vsadd_bu(dst1, src1); - dst0 = __lsx_vbitsel_v(dst0, alpha, control); - dst1 = __lsx_vbitsel_v(dst1, alpha, control); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - src_argb += 32; - src_argb1 += 32; - dst_argb += 32; - } +void ARGBShadeRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + int width, + uint32_t value) { + asm volatile( + "vreplgr2vr.w $vr4, %[val]\n\t" + "vilvl.b $vr5, $vr4, $vr4\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vilvl.b $vr1, $vr0, $vr0\n\t" + "vilvh.b $vr2, $vr0, $vr0\n\t" + "vmuh.hu $vr1, $vr1, $vr5\n\t" + "vmuh.hu $vr2, $vr2, $vr5\n\t" + "vsrlni.b.h $vr2, $vr1, 8\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [val] "r"(value) + : "memory", "$vr0", "$vr1", "$vr2", "$vr4", "$vr5"); +} + +void ARGBShuffleRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + const uint8_t* shuffler, + int width) { + asm volatile( + "vld $vr4, %[shuf], 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vshuf.b $vr0, $vr0, $vr0, $vr4\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [shuf] "r"(shuffler) + : "memory", "$vr0", "$vr4"); +} + +void ARGBGrayRow_LSX(const uint8_t* src_argb, uint8_t* dst_argb, int width) { + asm volatile( + "vldi $vr4, 29\n\t" + "vldi $vr5, 150\n\t" + "vldi $vr6, 77\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vld $vr1, %[src], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vmulwev.h.bu $vr7, $vr2, $vr4\n\t" + "vmaddwod.h.bu $vr7, $vr2, $vr6\n\t" + "vmaddwev.h.bu $vr7, $vr3, $vr5\n\t" + "vsrlri.h $vr7, $vr7, 8\n\t" + "vsrli.h $vr3, $vr3, 8\n\t" + "vslli.h $vr3, $vr3, 8\n\t" + "vor.v $vr3, $vr7, $vr3\n\t" + "vslli.h $vr8, $vr7, 8\n\t" + "vor.v $vr7, $vr7, $vr8\n\t" + "vilvl.h $vr0, $vr3, $vr7\n\t" + "vilvh.h $vr1, $vr3, $vr7\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8"); +} + +void ARGBSepiaRow_LSX(uint8_t* dst_argb, int width) { + asm volatile( + "vldi $vr4, 17\n\t" + "vldi $vr5, 68\n\t" + "vldi $vr6, 35\n\t" + "vldi $vr7, 22\n\t" + "vldi $vr8, 88\n\t" + "vldi $vr9, 45\n\t" + "vldi $vr10, 24\n\t" + "vldi $vr11, 98\n\t" + "vldi $vr12, 50\n\t" + "1:\n\t" + "vld $vr0, %[dst], 0\n\t" + "vld $vr1, %[dst], 16\n\t" + "vpickev.b $vr2, $vr1, $vr0\n\t" + "vpickod.b $vr3, $vr1, $vr0\n\t" + "vmulwev.h.bu $vr13, $vr2, $vr4\n\t" + "vmaddwod.h.bu $vr13, $vr2, $vr6\n\t" + "vmaddwev.h.bu $vr13, $vr3, $vr5\n\t" + "vmulwev.h.bu $vr14, $vr2, $vr7\n\t" + "vmaddwod.h.bu $vr14, $vr2, $vr9\n\t" + "vmaddwev.h.bu $vr14, $vr3, $vr8\n\t" + "vmulwev.h.bu $vr15, $vr2, $vr10\n\t" + "vmaddwod.h.bu $vr15, $vr2, $vr12\n\t" + "vmaddwev.h.bu $vr15, $vr3, $vr11\n\t" + "vsrli.h $vr13, $vr13, 7\n\t" + "vsrli.h $vr14, $vr14, 7\n\t" + "vsrli.h $vr15, $vr15, 7\n\t" + "vsat.hu $vr13, $vr13, 7\n\t" + "vsat.hu $vr14, $vr14, 7\n\t" + "vsat.hu $vr15, $vr15, 7\n\t" + "vpickev.b $vr13, $vr13, $vr13\n\t" + "vpickev.b $vr14, $vr14, $vr14\n\t" + "vpickev.b $vr15, $vr15, $vr15\n\t" + "vpickod.b $vr3, $vr3, $vr3\n\t" + "vilvl.b $vr2, $vr14, $vr13\n\t" + "vilvl.b $vr3, $vr3, $vr15\n\t" + "vilvl.h $vr0, $vr3, $vr2\n\t" + "vilvh.h $vr1, $vr3, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15"); +} + +void ARGBColorMatrixRow_LSX(const uint8_t* src_argb, + uint8_t* dst_argb, + const int8_t* matrix_argb, + int width) { + asm volatile( + "vldrepl.w $vr4, %[mat], 0\n\t" + "vldrepl.w $vr5, %[mat], 4\n\t" + "vldrepl.w $vr6, %[mat], 8\n\t" + "vldrepl.w $vr7, %[mat], 12\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vmulwev.h.bu.b $vr8, $vr0, $vr4\n\t" + "vmaddwod.h.bu.b $vr8, $vr0, $vr4\n\t" + "vmulwev.h.bu.b $vr9, $vr0, $vr5\n\t" + "vmaddwod.h.bu.b $vr9, $vr0, $vr5\n\t" + "vmulwev.h.bu.b $vr10, $vr0, $vr6\n\t" + "vmaddwod.h.bu.b $vr10, $vr0, $vr6\n\t" + "vmulwev.h.bu.b $vr11, $vr0, $vr7\n\t" + "vmaddwod.h.bu.b $vr11, $vr0, $vr7\n\t" + "vhaddw.w.h $vr8, $vr8, $vr8\n\t" + "vhaddw.w.h $vr9, $vr9, $vr9\n\t" + "vhaddw.w.h $vr10, $vr10, $vr10\n\t" + "vhaddw.w.h $vr11, $vr11, $vr11\n\t" + "vilvl.w $vr12, $vr9, $vr8\n\t" + "vilvh.w $vr13, $vr9, $vr8\n\t" + "vilvl.w $vr14, $vr11, $vr10\n\t" + "vilvh.w $vr15, $vr11, $vr10\n\t" + "vilvl.d $vr8, $vr14, $vr12\n\t" + "vilvh.d $vr9, $vr14, $vr12\n\t" + "vilvl.d $vr10, $vr15, $vr13\n\t" + "vilvh.d $vr11, $vr15, $vr13\n\t" + "vssrani.h.w $vr9, $vr8, 6\n\t" + "vssrani.h.w $vr11, $vr10, 6\n\t" + "vssrani.bu.h $vr11, $vr9, 0\n\t" + "vst $vr11, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [mat] "r"(matrix_argb) + : "memory", "$vr0", "$vr4", "$vr5", "$vr6", "$vr7", "$vr8", "$vr9", + "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15"); } void ARGBQuantizeRow_LSX(uint8_t* dst_argb, @@ -2398,333 +1852,875 @@ void ARGBQuantizeRow_LSX(uint8_t* dst_argb, int interval_size, int interval_offset, int width) { - int x; - int len = width / 16; - __m128i src0, src1, src2, src3, dst0, dst1, dst2, dst3; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; - __m128i vec_size = __lsx_vreplgr2vr_b(interval_size); - __m128i vec_offset = __lsx_vreplgr2vr_b(interval_offset); - __m128i vec_scale = __lsx_vreplgr2vr_w(scale); - __m128i zero = __lsx_vldi(0); - __m128i control = (__m128i)v2u64{0xFF000000FF000000, 0xFF000000FF000000}; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, dst_argb, 0, dst_argb, 16, dst_argb, 32, dst_argb, 48, - src0, src1, src2, src3); - reg0 = __lsx_vilvl_b(zero, src0); - reg1 = __lsx_vilvh_b(zero, src0); - reg2 = __lsx_vilvl_b(zero, src1); - reg3 = __lsx_vilvh_b(zero, src1); - reg4 = __lsx_vilvl_b(zero, src2); - reg5 = __lsx_vilvh_b(zero, src2); - reg6 = __lsx_vilvl_b(zero, src3); - reg7 = __lsx_vilvh_b(zero, src3); - tmp0 = __lsx_vilvl_h(zero, reg0); - tmp1 = __lsx_vilvh_h(zero, reg0); - tmp2 = __lsx_vilvl_h(zero, reg1); - tmp3 = __lsx_vilvh_h(zero, reg1); - tmp4 = __lsx_vilvl_h(zero, reg2); - tmp5 = __lsx_vilvh_h(zero, reg2); - tmp6 = __lsx_vilvl_h(zero, reg3); - tmp7 = __lsx_vilvh_h(zero, reg3); - DUP4_ARG2(__lsx_vmul_w, tmp0, vec_scale, tmp1, vec_scale, tmp2, vec_scale, - tmp3, vec_scale, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG2(__lsx_vmul_w, tmp4, vec_scale, tmp5, vec_scale, tmp6, vec_scale, - tmp7, vec_scale, tmp4, tmp5, tmp6, tmp7); - DUP4_ARG3(__lsx_vsrani_h_w, tmp1, tmp0, 16, tmp3, tmp2, 16, tmp5, tmp4, 16, - tmp7, tmp6, 16, reg0, reg1, reg2, reg3); - dst0 = __lsx_vpickev_b(reg1, reg0); - dst1 = __lsx_vpickev_b(reg3, reg2); - tmp0 = __lsx_vilvl_h(zero, reg4); - tmp1 = __lsx_vilvh_h(zero, reg4); - tmp2 = __lsx_vilvl_h(zero, reg5); - tmp3 = __lsx_vilvh_h(zero, reg5); - tmp4 = __lsx_vilvl_h(zero, reg6); - tmp5 = __lsx_vilvh_h(zero, reg6); - tmp6 = __lsx_vilvl_h(zero, reg7); - tmp7 = __lsx_vilvh_h(zero, reg7); - DUP4_ARG2(__lsx_vmul_w, tmp0, vec_scale, tmp1, vec_scale, tmp2, vec_scale, - tmp3, vec_scale, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG2(__lsx_vmul_w, tmp4, vec_scale, tmp5, vec_scale, tmp6, vec_scale, - tmp7, vec_scale, tmp4, tmp5, tmp6, tmp7); - DUP4_ARG3(__lsx_vsrani_h_w, tmp1, tmp0, 16, tmp3, tmp2, 16, tmp5, tmp4, 16, - tmp7, tmp6, 16, reg0, reg1, reg2, reg3); - dst2 = __lsx_vpickev_b(reg1, reg0); - dst3 = __lsx_vpickev_b(reg3, reg2); - DUP4_ARG2(__lsx_vmul_b, dst0, vec_size, dst1, vec_size, dst2, vec_size, - dst3, vec_size, dst0, dst1, dst2, dst3); - DUP4_ARG2(__lsx_vadd_b, dst0, vec_offset, dst1, vec_offset, dst2, - vec_offset, dst3, vec_offset, dst0, dst1, dst2, dst3); - DUP4_ARG3(__lsx_vbitsel_v, dst0, src0, control, dst1, src1, control, dst2, - src2, control, dst3, src3, control, dst0, dst1, dst2, dst3); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - __lsx_vst(dst2, dst_argb, 32); - __lsx_vst(dst3, dst_argb, 48); - dst_argb += 64; - } + asm volatile( + "vreplgr2vr.h $vr4, %[scl]\n\t" + "vreplgr2vr.b $vr5, %[isz]\n\t" + "vreplgr2vr.b $vr6, %[ioff]\n\t" + "vldi $vr7, 0xff\n\t" + "vslli.w $vr7, $vr7, 24\n\t" + "1:\n\t" + "vld $vr0, %[dst], 0\n\t" + "vilvl.b $vr1, $vr0, $vr0\n\t" + "vilvh.b $vr2, $vr0, $vr0\n\t" + "vsrli.h $vr1, $vr1, 8\n\t" + "vsrli.h $vr2, $vr2, 8\n\t" + "vmuh.hu $vr1, $vr1, $vr4\n\t" + "vmuh.hu $vr2, $vr2, $vr4\n\t" + "vsrlni.b.h $vr2, $vr1, 0\n\t" + "vmul.b $vr2, $vr2, $vr5\n\t" + "vadd.b $vr2, $vr2, $vr6\n\t" + "vand.v $vr0, $vr0, $vr7\n\t" + "vandn.v $vr2, $vr7, $vr2\n\t" + "vor.v $vr2, $vr2, $vr0\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -4\n\t" + "bnez %[cnt], 1b\n\t" + : [dst] "+r"(dst_argb), [cnt] "+r"(width) + : [scl] "r"(scale), [isz] "r"(interval_size), [ioff] "r"(interval_offset) + : "memory", "$vr0", "$vr1", "$vr2", "$vr4", "$vr5", "$vr6", "$vr7"); } -void ARGBColorMatrixRow_LSX(const uint8_t* src_argb, - uint8_t* dst_argb, - const int8_t* matrix_argb, - int width) { - int x; - int len = width / 8; - __m128i src0, src1, tmp0, tmp1, dst0, dst1; - __m128i tmp_b, tmp_g, tmp_r, tmp_a; - __m128i reg_b, reg_g, reg_r, reg_a; - __m128i matrix_b = __lsx_vldrepl_w(matrix_argb, 0); - __m128i matrix_g = __lsx_vldrepl_w(matrix_argb, 4); - __m128i matrix_r = __lsx_vldrepl_w(matrix_argb, 8); - __m128i matrix_a = __lsx_vldrepl_w(matrix_argb, 12); - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - DUP4_ARG2(__lsx_vdp2_h_bu_b, src0, matrix_b, src0, matrix_g, src0, matrix_r, - src0, matrix_a, tmp_b, tmp_g, tmp_r, tmp_a); - DUP4_ARG2(__lsx_vdp2_h_bu_b, src1, matrix_b, src1, matrix_g, src1, matrix_r, - src1, matrix_a, reg_b, reg_g, reg_r, reg_a); - DUP4_ARG2(__lsx_vhaddw_w_h, tmp_b, tmp_b, tmp_g, tmp_g, tmp_r, tmp_r, tmp_a, - tmp_a, tmp_b, tmp_g, tmp_r, tmp_a); - DUP4_ARG2(__lsx_vhaddw_w_h, reg_b, reg_b, reg_g, reg_g, reg_r, reg_r, reg_a, - reg_a, reg_b, reg_g, reg_r, reg_a); - DUP4_ARG2(__lsx_vsrai_w, tmp_b, 6, tmp_g, 6, tmp_r, 6, tmp_a, 6, tmp_b, - tmp_g, tmp_r, tmp_a); - DUP4_ARG2(__lsx_vsrai_w, reg_b, 6, reg_g, 6, reg_r, 6, reg_a, 6, reg_b, - reg_g, reg_r, reg_a); - DUP4_ARG1(__lsx_vclip255_w, tmp_b, tmp_g, tmp_r, tmp_a, tmp_b, tmp_g, tmp_r, - tmp_a) - DUP4_ARG1(__lsx_vclip255_w, reg_b, reg_g, reg_r, reg_a, reg_b, reg_g, reg_r, - reg_a) - DUP4_ARG2(__lsx_vpickev_h, reg_b, tmp_b, reg_g, tmp_g, reg_r, tmp_r, reg_a, - tmp_a, tmp_b, tmp_g, tmp_r, tmp_a); - tmp0 = __lsx_vpackev_b(tmp_g, tmp_b); - tmp1 = __lsx_vpackev_b(tmp_a, tmp_r); - dst0 = __lsx_vilvl_h(tmp1, tmp0); - dst1 = __lsx_vilvh_h(tmp1, tmp0); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - src_argb += 32; - dst_argb += 32; - } +void SobelRow_LSX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, + int width) { + asm volatile( + "vldi $vr4, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sx], 0\n\t" + "vldrepl.d $vr1, %[sy], 0\n\t" + "vsadd.bu $vr0, $vr0, $vr1\n\t" + "vilvl.b $vr2, $vr0, $vr0\n\t" + "vilvl.b $vr3, $vr4, $vr0\n\t" + "vilvl.h $vr0, $vr3, $vr2\n\t" + "vilvh.h $vr1, $vr3, $vr2\n\t" + "vst $vr0, %[dst], 0\n\t" + "vst $vr1, %[dst], 16\n\t" + "addi.d %[sx], %[sx], 8\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); } -void SplitUVRow_LSX(const uint8_t* src_uv, - uint8_t* dst_u, - uint8_t* dst_v, +void SobelToPlaneRow_LSX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_y, + int width) { + asm volatile( + "1:\n\t" + "vld $vr0, %[sx], 0\n\t" + "vld $vr1, %[sy], 0\n\t" + "vsadd.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[sx], %[sx], 16\n\t" + "addi.d %[sy], %[sy], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_y), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); +} + +void SobelXYRow_LSX(const uint8_t* src_sobelx, + const uint8_t* src_sobely, + uint8_t* dst_argb, int width) { - int x; - int len = width / 32; - __m128i src0, src1, src2, src3; - __m128i dst0, dst1, dst2, dst3; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_uv, 0, src_uv, 16, src_uv, 32, src_uv, 48, src0, - src1, src2, src3); - DUP2_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, dst0, dst1); - DUP2_ARG2(__lsx_vpickod_b, src1, src0, src3, src2, dst2, dst3); - __lsx_vst(dst0, dst_u, 0); - __lsx_vst(dst1, dst_u, 16); - __lsx_vst(dst2, dst_v, 0); - __lsx_vst(dst3, dst_v, 16); - src_uv += 64; - dst_u += 32; - dst_v += 32; - } + asm volatile( + "vldi $vr4, 0xff\n\t" + "1:\n\t" + "vldrepl.d $vr0, %[sx], 0\n\t" + "vldrepl.d $vr1, %[sy], 0\n\t" + "vsadd.bu $vr2, $vr0, $vr1\n\t" + "vilvl.b $vr1, $vr2, $vr1\n\t" + "vilvl.b $vr0, $vr4, $vr0\n\t" + "vilvl.h $vr2, $vr0, $vr1\n\t" + "vilvh.h $vr3, $vr0, $vr1\n\t" + "vst $vr2, %[dst], 0\n\t" + "vst $vr3, %[dst], 16\n\t" + "addi.d %[sx], %[sx], 8\n\t" + "addi.d %[sy], %[sy], 8\n\t" + "addi.d %[dst], %[dst], 32\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [sx] "+r"(src_sobelx), [sy] "+r"(src_sobely), [dst] "+r"(dst_argb), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); } -void SetRow_LSX(uint8_t* dst, uint8_t v8, int width) { - int x; - int len = width / 16; - __m128i dst0 = __lsx_vreplgr2vr_b(v8); - - for (x = 0; x < len; x++) { - __lsx_vst(dst0, dst, 0); - dst += 16; +void InterpolateRow_LSX(uint8_t* dst_ptr, + const uint8_t* src_ptr, + ptrdiff_t src_stride, + int width, + int32_t source_y_fraction) { + int y1_fraction = source_y_fraction; + const uint8_t* src_ptr1 = src_ptr + src_stride; + if (y1_fraction == 0) { + CopyRow_LSX(src_ptr, dst_ptr, width); + return; } -} - -void MirrorSplitUVRow_LSX(const uint8_t* src_uv, - uint8_t* dst_u, - uint8_t* dst_v, - int width) { - int x; - int len = width / 32; - __m128i src0, src1, src2, src3; - __m128i dst0, dst1, dst2, dst3; - __m128i shuff0 = {0x10121416181A1C1E, 0x00020406080A0C0E}; - __m128i shuff1 = {0x11131517191B1D1F, 0x01030507090B0D0F}; - - src_uv += (width << 1); - for (x = 0; x < len; x++) { - src_uv -= 64; - DUP4_ARG2(__lsx_vld, src_uv, 0, src_uv, 16, src_uv, 32, src_uv, 48, src2, - src3, src0, src1); - DUP4_ARG3(__lsx_vshuf_b, src1, src0, shuff1, src3, src2, shuff1, src1, src0, - shuff0, src3, src2, shuff0, dst0, dst1, dst2, dst3); - __lsx_vst(dst0, dst_v, 0); - __lsx_vst(dst1, dst_v, 16); - __lsx_vst(dst2, dst_u, 0); - __lsx_vst(dst3, dst_u, 16); - dst_u += 32; - dst_v += 32; + if (y1_fraction == 128) { + asm volatile( + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vavgr.bu $vr0, $vr0, $vr1\n\t" + "vst $vr0, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), + [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1"); + return; } + uint16_t y_frac = (uint16_t)((256 - y1_fraction) + (y1_fraction << 8)); + asm volatile( + "vreplgr2vr.h $vr3, %[yf]\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vld $vr1, %[s1], 0\n\t" + "vilvl.b $vr2, $vr1, $vr0\n\t" + "vilvh.b $vr4, $vr1, $vr0\n\t" + "vmulwev.h.bu $vr0, $vr2, $vr3\n\t" + "vmaddwod.h.bu $vr0, $vr2, $vr3\n\t" + "vmulwev.h.bu $vr1, $vr4, $vr3\n\t" + "vmaddwod.h.bu $vr1, $vr4, $vr3\n\t" + "vsrarni.b.h $vr1, $vr0, 8\n\t" + "vst $vr1, %[dst], 0\n\t" + "addi.d %[s0], %[s0], 16\n\t" + "addi.d %[s1], %[s1], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_ptr), [s1] "+r"(src_ptr1), [dst] "+r"(dst_ptr), + [cnt] "+r"(width) + : [yf] "r"(y_frac) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); } void HalfFloatRow_LSX(const uint16_t* src, uint16_t* dst, float scale, int width) { - int x; - int len = width / 32; float mult = 1.9259299444e-34f * scale; - __m128i src0, src1, src2, src3, dst0, dst1, dst2, dst3; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128 reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; - __m128 vec_mult = (__m128)__lsx_vldrepl_w(&mult, 0); - __m128i zero = __lsx_vldi(0); - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48, src0, src1, src2, - src3); - DUP4_ARG2(__lsx_vilvl_h, zero, src0, zero, src1, zero, src2, zero, src3, - tmp0, tmp2, tmp4, tmp6); - DUP4_ARG2(__lsx_vilvh_h, zero, src0, zero, src1, zero, src2, zero, src3, - tmp1, tmp3, tmp5, tmp7); - DUP4_ARG1(__lsx_vffint_s_wu, tmp0, tmp2, tmp4, tmp6, reg0, reg2, reg4, - reg6); - DUP4_ARG1(__lsx_vffint_s_wu, tmp1, tmp3, tmp5, tmp7, reg1, reg3, reg5, - reg7); - DUP4_ARG2(__lsx_vfmul_s, reg0, vec_mult, reg1, vec_mult, reg2, vec_mult, - reg3, vec_mult, reg0, reg1, reg2, reg3); - DUP4_ARG2(__lsx_vfmul_s, reg4, vec_mult, reg5, vec_mult, reg6, vec_mult, - reg7, vec_mult, reg4, reg5, reg6, reg7); - DUP4_ARG2(__lsx_vsrli_w, (v4u32)reg0, 13, (v4u32)reg1, 13, (v4u32)reg2, 13, - (v4u32)reg3, 13, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG2(__lsx_vsrli_w, (v4u32)reg4, 13, (v4u32)reg5, 13, (v4u32)reg6, 13, - (v4u32)reg7, 13, tmp4, tmp5, tmp6, tmp7); - DUP4_ARG2(__lsx_vpickev_h, tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, - dst0, dst1, dst2, dst3); - __lsx_vst(dst0, dst, 0); - __lsx_vst(dst1, dst, 16); - __lsx_vst(dst2, dst, 32); - __lsx_vst(dst3, dst, 48); - src += 32; - dst += 32; - } + asm volatile( + "vldrepl.w $vr3, %[mult], 0\n\t" + "vldi $vr4, 0\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vilvl.h $vr1, $vr4, $vr0\n\t" + "vilvh.h $vr2, $vr4, $vr0\n\t" + "vffint.s.wu $vr1, $vr1\n\t" + "vffint.s.wu $vr2, $vr2\n\t" + "vfmul.s $vr1, $vr1, $vr3\n\t" + "vfmul.s $vr2, $vr2, $vr3\n\t" + "vsrlni.h.w $vr2, $vr1, 13\n\t" + "vst $vr2, %[dst], 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src), [dst] "+r"(dst), [cnt] "+r"(width) + : [mult] "r"(&mult) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4"); } -// ARGB expects first 3 values to contain RGB and 4th value is ignored. void ARGBToYMatrixRow_LSX(const uint8_t* src_argb, uint8_t* dst_y, int width, const struct ArgbConstants* c) { asm volatile( - "vldrepl.b $vr0, %3, 0 \n\t" // load rgbconstants - "vldrepl.b $vr1, %3, 1 \n\t" // load rgbconstants - "vldrepl.b $vr2, %3, 2 \n\t" // load rgbconstants - "vldrepl.b $vr20, %3, 3 \n\t" // load rgbconstants - "vldrepl.h $vr3, %3, 96 \n\t" // load rgbconstants - "1: \n\t" - "vld $vr4, %0, 0 \n\t" - "vld $vr5, %0, 16 \n\t" - "vld $vr6, %0, 32 \n\t" - "vld $vr7, %0, 48 \n\t" // load 16 pixels of - // ARGB - "vor.v $vr12, $vr3, $vr3 \n\t" - "vor.v $vr13, $vr3, $vr3 \n\t" - "addi.d %2, %2, -16 \n\t" // 16 processed per - // loop. - "vpickev.b $vr8, $vr5, $vr4 \n\t" // BR - "vpickev.b $vr10, $vr7, $vr6 \n\t" - "vpickod.b $vr9, $vr5, $vr4 \n\t" // GA - "vpickod.b $vr11, $vr7, $vr6 \n\t" - "vmaddwev.h.bu $vr12, $vr8, $vr0 \n\t" // B - "vmaddwev.h.bu $vr13, $vr10, $vr0 \n\t" - "vmaddwev.h.bu $vr12, $vr9, $vr1 \n\t" // G - "vmaddwev.h.bu $vr13, $vr11, $vr1 \n\t" - "vmaddwod.h.bu $vr12, $vr8, $vr2 \n\t" // R - "vmaddwod.h.bu $vr13, $vr10, $vr2 \n\t" - "vmaddwod.h.bu $vr12, $vr9, $vr20 \n\t" // A - "vmaddwod.h.bu $vr13, $vr11, $vr20 \n\t" - "addi.d %0, %0, 64 \n\t" - "vpickod.b $vr10, $vr13, $vr12 \n\t" - "vst $vr10, %1, 0 \n\t" - "addi.d %1, %1, 16 \n\t" - "bnez %2, 1b \n\t" - : "+&r"(src_argb), // %0 - "+&r"(dst_y), // %1 - "+&r"(width) // %2 - : "r"(c) + "vldrepl.b $vr0, %[c], 0\n\t" + "vldrepl.b $vr1, %[c], 1\n\t" + "vldrepl.b $vr2, %[c], 2\n\t" + "vldrepl.b $vr3, %[c], 3\n\t" + "vldrepl.h $vr4, %[c], 96\n\t" + "1:\n\t" + "vld $vr5, %[src], 0\n\t" + "vld $vr6, %[src], 16\n\t" + "vor.v $vr9, $vr4, $vr4\n\t" + "vpickev.b $vr7, $vr6, $vr5\n\t" + "vpickod.b $vr8, $vr6, $vr5\n\t" + "vmaddwev.h.bu $vr9, $vr7, $vr0\n\t" + "vmaddwev.h.bu $vr9, $vr8, $vr1\n\t" + "vmaddwod.h.bu $vr9, $vr7, $vr2\n\t" + "vmaddwod.h.bu $vr9, $vr8, $vr3\n\t" + "vpickod.b $vr9, $vr9, $vr9\n\t" + "vstelm.d $vr9, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 32\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [c] "r"(c) : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr20"); + "$vr7", "$vr8", "$vr9"); } void RGBToYMatrixRow_LSX(const uint8_t* src_rgba, uint8_t* dst_y, int width, const struct ArgbConstants* c) { - static const int8_t shuff[64] = { - 0, 2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 17, 18, 20, 21, 23, - 24, 26, 27, 29, 30, 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, - 1, 0, 4, 0, 7, 0, 10, 0, 13, 0, 16, 0, 19, 0, 22, 0, - 25, 0, 28, 0, 31, 0, 2, 0, 5, 0, 8, 0, 11, 0, 14, 0, - }; + static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8, + 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8, + 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; asm volatile( - "vldrepl.b $vr0, %3, 0 \n\t" // load rgbconstants - "vldrepl.b $vr1, %3, 1 \n\t" // load rgbconstants - "vldrepl.b $vr2, %3, 2 \n\t" // load rgbconstants - "vldrepl.h $vr3, %3, 96 \n\t" // load rgbconstants - "vld $vr4, %4, 0 \n\t" // load shuff - "vld $vr5, %4, 16 \n\t" - "vld $vr6, %4, 32 \n\t" - "vld $vr7, %4, 48 \n\t" - "1: \n\t" - "vld $vr8, %0, 0 \n\t" - "vld $vr9, %0, 16 \n\t" - "vld $vr10, %0, 32 \n\t" // load 16 pixels of - // RGB - "vor.v $vr12, $vr3, $vr3 \n\t" - "vor.v $vr13, $vr3, $vr3 \n\t" - "addi.d %2, %2, -16 \n\t" // 16 processed per - // loop. - "vshuf.b $vr14, $vr9, $vr8, $vr4 \n\t" - "vshuf.b $vr15, $vr9, $vr10, $vr5 \n\t" - "vshuf.b $vr16, $vr9, $vr8, $vr6 \n\t" - "vshuf.b $vr17, $vr9, $vr10, $vr7 \n\t" - "vmaddwev.h.bu $vr12, $vr16, $vr1 \n\t" // G - "vmaddwev.h.bu $vr13, $vr17, $vr1 \n\t" - "vmaddwev.h.bu $vr12, $vr14, $vr0 \n\t" // B - "vmaddwev.h.bu $vr13, $vr15, $vr0 \n\t" - "vmaddwod.h.bu $vr12, $vr14, $vr2 \n\t" // R - "vmaddwod.h.bu $vr13, $vr15, $vr2 \n\t" - "addi.d %0, %0, 48 \n\t" - "vpickod.b $vr10, $vr13, $vr12 \n\t" - "vst $vr10, %1, 0 \n\t" - "addi.d %1, %1, 16 \n\t" - "bnez %2, 1b \n\t" - : "+&r"(src_rgba), // %0 - "+&r"(dst_y), // %1 - "+&r"(width) // %2 - : "r"(c), // %3 - "r"(shuff) // %4 + "vldrepl.b $vr0, %[c], 0\n\t" + "vldrepl.b $vr1, %[c], 1\n\t" + "vldrepl.b $vr2, %[c], 2\n\t" + "vldrepl.h $vr3, %[c], 96\n\t" + "vld $vr4, %[shuf], 0\n\t" + "vld $vr5, %[shuf], 16\n\t" + "1:\n\t" + "vld $vr6, %[src], 0\n\t" + "vld $vr7, %[src], 12\n\t" + "vld $vr8, %[src], 24\n\t" + "vld $vr9, %[src], 32\n\t" + "vshuf.b $vr6, $vr6, $vr6, $vr4\n\t" + "vshuf.b $vr7, $vr7, $vr7, $vr4\n\t" + "vshuf.b $vr8, $vr8, $vr8, $vr4\n\t" + "vshuf.b $vr9, $vr9, $vr9, $vr5\n\t" + "vor.v $vr12, $vr3, $vr3\n\t" + "vor.v $vr13, $vr3, $vr3\n\t" + "vpickev.b $vr10, $vr7, $vr6\n\t" + "vpickod.b $vr11, $vr7, $vr6\n\t" + "vmaddwev.h.bu $vr12, $vr10, $vr0\n\t" + "vmaddwev.h.bu $vr12, $vr11, $vr1\n\t" + "vmaddwod.h.bu $vr12, $vr10, $vr2\n\t" + "vpickev.b $vr10, $vr9, $vr8\n\t" + "vpickod.b $vr11, $vr9, $vr8\n\t" + "vmaddwev.h.bu $vr13, $vr10, $vr0\n\t" + "vmaddwev.h.bu $vr13, $vr11, $vr1\n\t" + "vmaddwod.h.bu $vr13, $vr10, $vr2\n\t" + "vpickod.b $vr12, $vr13, $vr12\n\t" + "vst $vr12, %[dst], 0\n\t" + "addi.d %[src], %[src], 48\n\t" + "addi.d %[dst], %[dst], 16\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgba), [dst] "+r"(dst_y), [cnt] "+r"(width) + : [c] "r"(c), [shuf] "r"(kShuf) : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", - "$vr7", "$vr8", "$vr9", "$vr10", "$vr12", "$vr13", "$vr14", "$vr15", - "$vr16", "$vr17"); + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13"); } -// undef for unified sources build -#undef YUVTORGB_SETUP -#undef READYUV422_D -#undef READYUV422 -#undef YUVTORGB_D -#undef YUVTORGB -#undef I444TORGB -#undef STOREARGB_D -#undef STOREARGB -#undef RGBTOUV +void RGB565ToYRow_LSX(const uint8_t* src_rgb565, uint8_t* dst_y, int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "vldi $vr8, 25\n\t" + "vsrli.h $vr8, $vr8, 8\n\t" + "vldi $vr9, 129\n\t" + "vsrli.h $vr9, $vr9, 8\n\t" + "vldi $vr10, 66\n\t" + "vsrli.h $vr10, $vr10, 8\n\t" + "vldi $vr11, 33\n\t" + "vsrli.h $vr11, $vr11, 8\n\t" + "vslli.h $vr11, $vr11, 7\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vmul.h $vr1, $vr1, $vr8\n\t" + "vmadd.h $vr1, $vr2, $vr9\n\t" + "vmadd.h $vr1, $vr3, $vr10\n\t" + "vadd.h $vr1, $vr1, $vr11\n\t" + "vpickod.b $vr1, $vr1, $vr1\n\t" + "vstelm.d $vr1, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_rgb565), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr6", "$vr7", + "$vr8", "$vr9", "$vr10", "$vr11"); +} + +void ARGB1555ToYRow_LSX(const uint8_t* src_argb1555, + uint8_t* dst_y, + int width) { + asm volatile( + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr8, 25\n\t" + "vsrli.h $vr8, $vr8, 8\n\t" + "vldi $vr9, 129\n\t" + "vsrli.h $vr9, $vr9, 8\n\t" + "vldi $vr10, 66\n\t" + "vsrli.h $vr10, $vr10, 8\n\t" + "vldi $vr11, 33\n\t" + "vsrli.h $vr11, $vr11, 8\n\t" + "vslli.h $vr11, $vr11, 7\n\t" + "1:\n\t" + "vld $vr0, %[src], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vsrai.h $vr5, $vr0, 15\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vmul.h $vr1, $vr1, $vr8\n\t" + "vmadd.h $vr1, $vr2, $vr9\n\t" + "vmadd.h $vr1, $vr3, $vr10\n\t" + "vadd.h $vr1, $vr1, $vr11\n\t" + "vpickod.b $vr1, $vr1, $vr1\n\t" + "vstelm.d $vr1, %[dst], 0, 0\n\t" + "addi.d %[src], %[src], 16\n\t" + "addi.d %[dst], %[dst], 8\n\t" + "addi.d %[cnt], %[cnt], -8\n\t" + "bnez %[cnt], 1b\n\t" + : [src] "+r"(src_argb1555), [dst] "+r"(dst_y), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr8", "$vr9", "$vr10", "$vr11"); +} + +void RGB565ToUVRow_LSX(const uint8_t* src_rgb565, + int src_stride_rgb565, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_rgb565 = src_rgb565 + src_stride_rgb565; + asm volatile( + "vldi $vr20, 112\n\t" + "vsrli.h $vr20, $vr20, 8\n\t" + "vldi $vr21, 74\n\t" + "vsrli.h $vr21, $vr21, 8\n\t" + "vldi $vr22, 38\n\t" + "vsrli.h $vr22, $vr22, 8\n\t" + "vldi $vr23, 94\n\t" + "vsrli.h $vr23, $vr23, 8\n\t" + "vldi $vr24, 18\n\t" + "vsrli.h $vr24, $vr24, 8\n\t" + "vldi $vr25, 128\n\t" + "vsrli.h $vr25, $vr25, 8\n\t" + "vslli.h $vr25, $vr25, 8\n\t" + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vor.v $vr10, $vr1, $vr1\n\t" + "vor.v $vr11, $vr2, $vr2\n\t" + "vor.v $vr12, $vr3, $vr3\n\t" + "vld $vr0, %[s0], 16\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr10, $vr1, $vr10\n\t" + "vpickev.b $vr11, $vr2, $vr11\n\t" + "vpickev.b $vr12, $vr3, $vr12\n\t" + "vld $vr0, %[s1], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vor.v $vr13, $vr1, $vr1\n\t" + "vor.v $vr14, $vr2, $vr2\n\t" + "vor.v $vr15, $vr3, $vr3\n\t" + "vld $vr0, %[s1], 16\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr7\n\t" + "vsrli.h $vr3, $vr0, 11\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 2\n\t" + "vsrli.h $vr2, $vr2, 4\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr5, $vr3, $vr15\n\t" + "vpickev.b $vr4, $vr2, $vr14\n\t" + "vpickev.b $vr3, $vr1, $vr13\n\t" + "vor.v $vr0, $vr10, $vr10\n\t" + "vor.v $vr1, $vr11, $vr11\n\t" + "vor.v $vr2, $vr12, $vr12\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr3\n\t" + "vsrari.h $vr0, $vr0, 2\n\t" + "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" + "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" + "vadd.h $vr1, $vr1, $vr4\n\t" + "vsrari.h $vr1, $vr1, 2\n\t" + "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" + "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr5\n\t" + "vsrari.h $vr2, $vr2, 2\n\t" + "vor.v $vr6, $vr25, $vr25\n\t" + "vor.v $vr7, $vr25, $vr25\n\t" + "vmadd.h $vr6, $vr0, $vr20\n\t" + "vmsub.h $vr6, $vr1, $vr21\n\t" + "vmsub.h $vr6, $vr2, $vr22\n\t" + "vmadd.h $vr7, $vr2, $vr20\n\t" + "vmsub.h $vr7, $vr1, $vr23\n\t" + "vmsub.h $vr7, $vr0, $vr24\n\t" + "vpickod.b $vr6, $vr7, $vr6\n\t" + "vstelm.d $vr6, %[du], 0, 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "vldi $vr7, 63\n\t" + "vsrli.h $vr7, $vr7, 8\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_rgb565), [s1] "+r"(next_rgb565), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15", "$vr20", + "$vr21", "$vr22", "$vr23", "$vr24", "$vr25"); +} + +void ARGB1555ToUVRow_LSX(const uint8_t* src_argb1555, + int src_stride_argb1555, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + const uint8_t* next_argb1555 = src_argb1555 + src_stride_argb1555; + asm volatile( + "vldi $vr20, 112\n\t" + "vsrli.h $vr20, $vr20, 8\n\t" + "vldi $vr21, 74\n\t" + "vsrli.h $vr21, $vr21, 8\n\t" + "vldi $vr22, 38\n\t" + "vsrli.h $vr22, $vr22, 8\n\t" + "vldi $vr23, 94\n\t" + "vsrli.h $vr23, $vr23, 8\n\t" + "vldi $vr24, 18\n\t" + "vsrli.h $vr24, $vr24, 8\n\t" + "vldi $vr25, 128\n\t" + "vsrli.h $vr25, $vr25, 8\n\t" + "vslli.h $vr25, $vr25, 8\n\t" + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "1:\n\t" + "vld $vr0, %[s0], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vsrai.h $vr5, $vr0, 15\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vor.v $vr10, $vr1, $vr1\n\t" + "vor.v $vr11, $vr2, $vr2\n\t" + "vor.v $vr12, $vr3, $vr3\n\t" + "vld $vr0, %[s0], 16\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vsrai.h $vr5, $vr0, 15\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr10, $vr1, $vr10\n\t" + "vpickev.b $vr11, $vr2, $vr11\n\t" + "vpickev.b $vr12, $vr3, $vr12\n\t" + "vld $vr0, %[s1], 0\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vsrai.h $vr5, $vr0, 15\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vor.v $vr13, $vr1, $vr1\n\t" + "vor.v $vr14, $vr2, $vr2\n\t" + "vor.v $vr15, $vr3, $vr3\n\t" + "vld $vr0, %[s1], 16\n\t" + "vand.v $vr1, $vr0, $vr6\n\t" + "vsrli.h $vr2, $vr0, 5\n\t" + "vand.v $vr2, $vr2, $vr6\n\t" + "vsrli.h $vr3, $vr0, 10\n\t" + "vand.v $vr3, $vr3, $vr6\n\t" + "vsrai.h $vr5, $vr0, 15\n\t" + "vslli.h $vr4, $vr1, 3\n\t" + "vsrli.h $vr1, $vr1, 2\n\t" + "vor.v $vr1, $vr4, $vr1\n\t" + "vslli.h $vr4, $vr2, 3\n\t" + "vsrli.h $vr2, $vr2, 2\n\t" + "vor.v $vr2, $vr4, $vr2\n\t" + "vslli.h $vr4, $vr3, 3\n\t" + "vsrli.h $vr3, $vr3, 2\n\t" + "vor.v $vr3, $vr4, $vr3\n\t" + "vpickev.b $vr5, $vr3, $vr15\n\t" + "vpickev.b $vr4, $vr2, $vr14\n\t" + "vpickev.b $vr3, $vr1, $vr13\n\t" + "vor.v $vr0, $vr10, $vr10\n\t" + "vor.v $vr1, $vr11, $vr11\n\t" + "vor.v $vr2, $vr12, $vr12\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr3\n\t" + "vsrari.h $vr0, $vr0, 2\n\t" + "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" + "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" + "vadd.h $vr1, $vr1, $vr4\n\t" + "vsrari.h $vr1, $vr1, 2\n\t" + "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" + "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr5\n\t" + "vsrari.h $vr2, $vr2, 2\n\t" + "vor.v $vr6, $vr25, $vr25\n\t" + "vor.v $vr7, $vr25, $vr25\n\t" + "vmadd.h $vr6, $vr0, $vr20\n\t" + "vmsub.h $vr6, $vr1, $vr21\n\t" + "vmsub.h $vr6, $vr2, $vr22\n\t" + "vmadd.h $vr7, $vr2, $vr20\n\t" + "vmsub.h $vr7, $vr1, $vr23\n\t" + "vmsub.h $vr7, $vr0, $vr24\n\t" + "vpickod.b $vr6, $vr7, $vr6\n\t" + "vstelm.d $vr6, %[du], 0, 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "vldi $vr6, 31\n\t" + "vsrli.h $vr6, $vr6, 8\n\t" + "addi.d %[s0], %[s0], 32\n\t" + "addi.d %[s1], %[s1], 32\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src_argb1555), [s1] "+r"(next_argb1555), [du] "+r"(dst_u), + [dv] "+r"(dst_v), [cnt] "+r"(width) + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", "$vr15", "$vr20", + "$vr21", "$vr22", "$vr23", "$vr24", "$vr25"); +} + +static LIBYUV_NOINLINE void XRGBToUVHelper_LSX(const uint8_t* src, + int stride, + uint8_t* dst_u, + uint8_t* dst_v, + const uint8_t* shuf, + int width) { + const uint8_t* next = src + stride; + asm volatile( + "vldi $vr20, 112\n\t" + "vsrli.h $vr20, $vr20, 8\n\t" + "vldi $vr21, 74\n\t" + "vsrli.h $vr21, $vr21, 8\n\t" + "vldi $vr22, 38\n\t" + "vsrli.h $vr22, $vr22, 8\n\t" + "vldi $vr23, 94\n\t" + "vsrli.h $vr23, $vr23, 8\n\t" + "vldi $vr24, 18\n\t" + "vsrli.h $vr24, $vr24, 8\n\t" + "vldi $vr25, 128\n\t" + "vsrli.h $vr25, $vr25, 8\n\t" + "vslli.h $vr25, $vr25, 8\n\t" + "vld $vr16, %[shuf], 0\n\t" + "1:\n\t" + "vld $vr8, %[s0], 0\n\t" + "vld $vr9, %[s0], 16\n\t" + "vld $vr10, %[s0], 32\n\t" + "vld $vr11, %[s0], 48\n\t" + "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" + "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" + "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" + "vshuf.b $vr11, $vr11, $vr11, $vr16\n\t" + "vpickev.b $vr12, $vr9, $vr8\n\t" + "vpickod.b $vr13, $vr9, $vr8\n\t" + "vpickev.b $vr14, $vr11, $vr10\n\t" + "vpickod.b $vr15, $vr11, $vr10\n\t" + "vpickev.b $vr0, $vr14, $vr12\n\t" + "vpickod.b $vr2, $vr14, $vr12\n\t" + "vpickev.b $vr1, $vr15, $vr13\n\t" + "vld $vr8, %[s1], 0\n\t" + "vld $vr9, %[s1], 16\n\t" + "vld $vr10, %[s1], 32\n\t" + "vld $vr11, %[s1], 48\n\t" + "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" + "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" + "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" + "vshuf.b $vr11, $vr11, $vr11, $vr16\n\t" + "vpickev.b $vr12, $vr9, $vr8\n\t" + "vpickod.b $vr13, $vr9, $vr8\n\t" + "vpickev.b $vr14, $vr11, $vr10\n\t" + "vpickod.b $vr15, $vr11, $vr10\n\t" + "vpickev.b $vr3, $vr14, $vr12\n\t" + "vpickod.b $vr5, $vr14, $vr12\n\t" + "vpickev.b $vr4, $vr15, $vr13\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr3\n\t" + "vsrari.h $vr0, $vr0, 2\n\t" + "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" + "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" + "vadd.h $vr1, $vr1, $vr4\n\t" + "vsrari.h $vr1, $vr1, 2\n\t" + "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" + "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr5\n\t" + "vsrari.h $vr2, $vr2, 2\n\t" + "vor.v $vr6, $vr25, $vr25\n\t" + "vor.v $vr7, $vr25, $vr25\n\t" + "vmadd.h $vr6, $vr0, $vr20\n\t" + "vmsub.h $vr6, $vr1, $vr21\n\t" + "vmsub.h $vr6, $vr2, $vr22\n\t" + "vmadd.h $vr7, $vr2, $vr20\n\t" + "vmsub.h $vr7, $vr1, $vr23\n\t" + "vmsub.h $vr7, $vr0, $vr24\n\t" + "vpickod.b $vr6, $vr7, $vr6\n\t" + "vstelm.d $vr6, %[du], 0, 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "addi.d %[s0], %[s0], 64\n\t" + "addi.d %[s1], %[s1], 64\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15", "$vr16", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24", "$vr25"); +} + +void BGRAToUVRow_LSX(const uint8_t* src_bgra, + int src_stride_bgra, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[16] = {3, 2, 1, 0, 7, 6, 5, 4, + 11, 10, 9, 8, 15, 14, 13, 12}; + XRGBToUVHelper_LSX(src_bgra, src_stride_bgra, dst_u, dst_v, kShuf, width); +} + +void ABGRToUVRow_LSX(const uint8_t* src_abgr, + int src_stride_abgr, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[16] = {2, 1, 0, 3, 6, 5, 4, 7, + 10, 9, 8, 11, 14, 13, 12, 15}; + XRGBToUVHelper_LSX(src_abgr, src_stride_abgr, dst_u, dst_v, kShuf, width); +} + +void RGBAToUVRow_LSX(const uint8_t* src_rgba, + int src_stride_rgba, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[16] = {1, 2, 3, 0, 5, 6, 7, 4, + 9, 10, 11, 8, 13, 14, 15, 12}; + XRGBToUVHelper_LSX(src_rgba, src_stride_rgba, dst_u, dst_v, kShuf, width); +} + +static LIBYUV_NOINLINE void RGB24ToUVHelper_LSX(const uint8_t* src, + int stride, + uint8_t* dst_u, + uint8_t* dst_v, + const uint8_t* shuf, + int width) { + const uint8_t* next = src + stride; + asm volatile( + "vldi $vr20, 112\n\t" + "vsrli.h $vr20, $vr20, 8\n\t" + "vldi $vr21, 74\n\t" + "vsrli.h $vr21, $vr21, 8\n\t" + "vldi $vr22, 38\n\t" + "vsrli.h $vr22, $vr22, 8\n\t" + "vldi $vr23, 94\n\t" + "vsrli.h $vr23, $vr23, 8\n\t" + "vldi $vr24, 18\n\t" + "vsrli.h $vr24, $vr24, 8\n\t" + "vldi $vr25, 128\n\t" + "vsrli.h $vr25, $vr25, 8\n\t" + "vslli.h $vr25, $vr25, 8\n\t" + "vld $vr16, %[shuf], 0\n\t" + "vld $vr17, %[shuf], 16\n\t" + "1:\n\t" + "vld $vr8, %[s0], 0\n\t" + "vld $vr9, %[s0], 12\n\t" + "vld $vr10, %[s0], 24\n\t" + "vld $vr11, %[s0], 32\n\t" + "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" + "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" + "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" + "vshuf.b $vr11, $vr11, $vr11, $vr17\n\t" + "vpickev.b $vr12, $vr9, $vr8\n\t" + "vpickod.b $vr13, $vr9, $vr8\n\t" + "vpickev.b $vr14, $vr11, $vr10\n\t" + "vpickod.b $vr15, $vr11, $vr10\n\t" + "vpickev.b $vr0, $vr14, $vr12\n\t" + "vpickod.b $vr2, $vr14, $vr12\n\t" + "vpickev.b $vr1, $vr15, $vr13\n\t" + "vld $vr8, %[s1], 0\n\t" + "vld $vr9, %[s1], 12\n\t" + "vld $vr10, %[s1], 24\n\t" + "vld $vr11, %[s1], 32\n\t" + "vshuf.b $vr8, $vr8, $vr8, $vr16\n\t" + "vshuf.b $vr9, $vr9, $vr9, $vr16\n\t" + "vshuf.b $vr10, $vr10, $vr10, $vr16\n\t" + "vshuf.b $vr11, $vr11, $vr11, $vr17\n\t" + "vpickev.b $vr12, $vr9, $vr8\n\t" + "vpickod.b $vr13, $vr9, $vr8\n\t" + "vpickev.b $vr14, $vr11, $vr10\n\t" + "vpickod.b $vr15, $vr11, $vr10\n\t" + "vpickev.b $vr3, $vr14, $vr12\n\t" + "vpickod.b $vr5, $vr14, $vr12\n\t" + "vpickev.b $vr4, $vr15, $vr13\n\t" + "vhaddw.hu.bu $vr0, $vr0, $vr0\n\t" + "vhaddw.hu.bu $vr3, $vr3, $vr3\n\t" + "vadd.h $vr0, $vr0, $vr3\n\t" + "vsrari.h $vr0, $vr0, 2\n\t" + "vhaddw.hu.bu $vr1, $vr1, $vr1\n\t" + "vhaddw.hu.bu $vr4, $vr4, $vr4\n\t" + "vadd.h $vr1, $vr1, $vr4\n\t" + "vsrari.h $vr1, $vr1, 2\n\t" + "vhaddw.hu.bu $vr2, $vr2, $vr2\n\t" + "vhaddw.hu.bu $vr5, $vr5, $vr5\n\t" + "vadd.h $vr2, $vr2, $vr5\n\t" + "vsrari.h $vr2, $vr2, 2\n\t" + "vor.v $vr6, $vr25, $vr25\n\t" + "vor.v $vr7, $vr25, $vr25\n\t" + "vmadd.h $vr6, $vr0, $vr20\n\t" + "vmsub.h $vr6, $vr1, $vr21\n\t" + "vmsub.h $vr6, $vr2, $vr22\n\t" + "vmadd.h $vr7, $vr2, $vr20\n\t" + "vmsub.h $vr7, $vr1, $vr23\n\t" + "vmsub.h $vr7, $vr0, $vr24\n\t" + "vpickod.b $vr6, $vr7, $vr6\n\t" + "vstelm.d $vr6, %[du], 0, 0\n\t" + "vstelm.d $vr6, %[dv], 0, 1\n\t" + "addi.d %[s0], %[s0], 48\n\t" + "addi.d %[s1], %[s1], 48\n\t" + "addi.d %[du], %[du], 8\n\t" + "addi.d %[dv], %[dv], 8\n\t" + "addi.d %[cnt], %[cnt], -16\n\t" + "bnez %[cnt], 1b\n\t" + : [s0] "+r"(src), [s1] "+r"(next), [du] "+r"(dst_u), [dv] "+r"(dst_v), + [cnt] "+r"(width) + : [shuf] "r"(shuf) + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11", "$vr12", "$vr13", "$vr14", + "$vr15", "$vr16", "$vr17", "$vr20", "$vr21", "$vr22", "$vr23", "$vr24", + "$vr25"); +} + +void RGB24ToUVRow_LSX(const uint8_t* src_rgb24, + int src_stride_rgb24, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[32] = {0, 1, 2, 0, 3, 4, 5, 0, 6, 7, 8, + 0, 9, 10, 11, 0, 4, 5, 6, 0, 7, 8, + 9, 0, 10, 11, 12, 0, 13, 14, 15, 0}; + RGB24ToUVHelper_LSX(src_rgb24, src_stride_rgb24, dst_u, dst_v, kShuf, width); +} + +void RAWToUVRow_LSX(const uint8_t* src_raw, + int src_stride_raw, + uint8_t* dst_u, + uint8_t* dst_v, + int width) { + static const uint8_t kShuf[32] = {2, 1, 0, 0, 5, 4, 3, 0, 8, 7, 6, + 0, 11, 10, 9, 0, 6, 5, 4, 0, 9, 8, + 7, 0, 12, 11, 10, 0, 15, 14, 13, 0}; + RGB24ToUVHelper_LSX(src_raw, src_stride_raw, dst_u, dst_v, kShuf, width); +} #ifdef __cplusplus } // extern "C" diff --git a/source/scale.cc b/source/scale.cc index ba06d42ea..668c3d4b7 100644 --- a/source/scale.cc +++ b/source/scale.cc @@ -130,7 +130,7 @@ static void ScalePlaneDown2(int src_width, ? ScaleRowDown2_Any_LSX : (filtering == kFilterLinear ? ScaleRowDown2Linear_Any_LSX : ScaleRowDown2Box_Any_LSX); - if (IS_ALIGNED(dst_width, 32)) { + if (IS_ALIGNED(dst_width, 16)) { ScaleRowDown2 = filtering == kFilterNone ? ScaleRowDown2_LSX : (filtering == kFilterLinear ? ScaleRowDown2Linear_LSX @@ -288,7 +288,7 @@ static void ScalePlaneDown4(int src_width, if (TestCpuFlag(kCpuHasLSX)) { ScaleRowDown4 = filtering ? ScaleRowDown4Box_Any_LSX : ScaleRowDown4_Any_LSX; - if (IS_ALIGNED(dst_width, 16)) { + if (IS_ALIGNED(dst_width, 8)) { ScaleRowDown4 = filtering ? ScaleRowDown4Box_LSX : ScaleRowDown4_LSX; } } @@ -415,7 +415,7 @@ static void ScalePlaneDown34(int src_width, #endif #if defined(HAS_SCALEROWDOWN34_LSX) if (TestCpuFlag(kCpuHasLSX)) { - if (dst_width % 48 == 0) { + if (dst_width % 12 == 0) { if (!filtering) { ScaleRowDown34_0 = ScaleRowDown34_LSX; ScaleRowDown34_1 = ScaleRowDown34_LSX; @@ -1155,11 +1155,19 @@ static int ScalePlaneBilinearDown(int src_width, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(src_width, 32)) { + if (IS_ALIGNED(src_width, 16)) { InterpolateRow = InterpolateRow_LSX; } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(src_width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; @@ -1190,7 +1198,7 @@ static int ScalePlaneBilinearDown(int src_width, #if defined(HAS_SCALEFILTERCOLS_LSX) if (TestCpuFlag(kCpuHasLSX) && src_width < 32768) { ScaleFilterCols = ScaleFilterCols_Any_LSX; - if (IS_ALIGNED(dst_width, 16)) { + if (IS_ALIGNED(dst_width, 4)) { ScaleFilterCols = ScaleFilterCols_LSX; } } @@ -1423,7 +1431,7 @@ static int ScalePlaneBilinearUp(int src_width, #if defined(HAS_SCALEFILTERCOLS_LSX) if (filtering && TestCpuFlag(kCpuHasLSX) && src_width < 32768) { ScaleFilterCols = ScaleFilterCols_Any_LSX; - if (IS_ALIGNED(dst_width, 16)) { + if (IS_ALIGNED(dst_width, 4)) { ScaleFilterCols = ScaleFilterCols_LSX; } } diff --git a/source/scale_any.cc b/source/scale_any.cc index 8d17e62c7..3062ee474 100644 --- a/source/scale_any.cc +++ b/source/scale_any.cc @@ -154,19 +154,19 @@ SDANY(ScaleUVRowDown2Box_Any_NEON, 7) #endif #ifdef HAS_SCALEROWDOWN2_LSX -SDANY(ScaleRowDown2_Any_LSX, ScaleRowDown2_LSX, ScaleRowDown2_C, 2, 1, 31) +SDANY(ScaleRowDown2_Any_LSX, ScaleRowDown2_LSX, ScaleRowDown2_C, 2, 1, 15) SDANY(ScaleRowDown2Linear_Any_LSX, ScaleRowDown2Linear_LSX, ScaleRowDown2Linear_C, 2, 1, - 31) + 15) SDANY(ScaleRowDown2Box_Any_LSX, ScaleRowDown2Box_LSX, ScaleRowDown2Box_C, 2, 1, - 31) + 15) #endif #ifdef HAS_SCALEROWDOWN4_SSSE3 SDANY(ScaleRowDown4_Any_SSSE3, ScaleRowDown4_SSSE3, ScaleRowDown4_C, 4, 1, 7) @@ -196,13 +196,13 @@ SDANY(ScaleRowDown4Box_Any_NEON, 7) #endif #ifdef HAS_SCALEROWDOWN4_LSX -SDANY(ScaleRowDown4_Any_LSX, ScaleRowDown4_LSX, ScaleRowDown4_C, 4, 1, 15) +SDANY(ScaleRowDown4_Any_LSX, ScaleRowDown4_LSX, ScaleRowDown4_C, 4, 1, 7) SDANY(ScaleRowDown4Box_Any_LSX, ScaleRowDown4Box_LSX, ScaleRowDown4Box_C, 4, 1, - 15) + 7) #endif #ifdef HAS_SCALEROWDOWN34_SSSE3 SDANY(ScaleRowDown34_Any_SSSE3, @@ -271,19 +271,19 @@ SDANY(ScaleRowDown34_Any_LSX, ScaleRowDown34_C, 4 / 3, 1, - 47) + 11) SDANY(ScaleRowDown34_0_Box_Any_LSX, ScaleRowDown34_0_Box_LSX, ScaleRowDown34_0_Box_C, 4 / 3, 1, - 47) + 11) SDANY(ScaleRowDown34_1_Box_Any_LSX, ScaleRowDown34_1_Box_LSX, ScaleRowDown34_1_Box_C, 4 / 3, 1, - 47) + 11) #endif #ifdef HAS_SCALEROWDOWN38_SSSE3 SDANY(ScaleRowDown38_Any_SSSE3, @@ -669,7 +669,7 @@ SAANY(ScaleAddRow_Any_LSX, ScaleAddRow_LSX, ScaleAddRow_C, 15) CANY(ScaleFilterCols_Any_NEON, ScaleFilterCols_NEON, ScaleFilterCols_C, 1, 7) #endif #ifdef HAS_SCALEFILTERCOLS_LSX -CANY(ScaleFilterCols_Any_LSX, ScaleFilterCols_LSX, ScaleFilterCols_C, 1, 15) +CANY(ScaleFilterCols_Any_LSX, ScaleFilterCols_LSX, ScaleFilterCols_C, 1, 3) #endif #ifdef HAS_SCALEARGBCOLS_NEON CANY(ScaleARGBCols_Any_NEON, ScaleARGBCols_NEON, ScaleARGBCols_C, 4, 7) @@ -689,7 +689,7 @@ CANY(ScaleARGBFilterCols_Any_LSX, ScaleARGBFilterCols_LSX, ScaleARGBFilterCols_C, 4, - 7) + 3) #endif #ifdef HAS_SCALEFILTERCOLS_WASMSIMD CANY(ScaleFilterCols_Any_WASMSIMD, diff --git a/source/scale_argb.cc b/source/scale_argb.cc index a1650dc72..9ed8f198f 100644 --- a/source/scale_argb.cc +++ b/source/scale_argb.cc @@ -399,11 +399,19 @@ static int ScaleARGBBilinearDown(int src_width, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(clip_src_width, 32)) { + if (IS_ALIGNED(clip_src_width, 16)) { InterpolateRow = InterpolateRow_LSX; } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(clip_src_width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; @@ -433,7 +441,7 @@ static int ScaleARGBBilinearDown(int src_width, #if defined(HAS_SCALEARGBFILTERCOLS_LSX) if (TestCpuFlag(kCpuHasLSX)) { ScaleARGBFilterCols = ScaleARGBFilterCols_Any_LSX; - if (IS_ALIGNED(dst_width, 8)) { + if (IS_ALIGNED(dst_width, 4)) { ScaleARGBFilterCols = ScaleARGBFilterCols_LSX; } } @@ -544,11 +552,19 @@ static int ScaleARGBBilinearUp(int src_width, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(dst_width, 8)) { + if (IS_ALIGNED(dst_width, 16)) { InterpolateRow = InterpolateRow_LSX; } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(dst_width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; @@ -582,7 +598,7 @@ static int ScaleARGBBilinearUp(int src_width, #if defined(HAS_SCALEARGBFILTERCOLS_LSX) if (filtering && TestCpuFlag(kCpuHasLSX)) { ScaleARGBFilterCols = ScaleARGBFilterCols_Any_LSX; - if (IS_ALIGNED(dst_width, 8)) { + if (IS_ALIGNED(dst_width, 4)) { ScaleARGBFilterCols = ScaleARGBFilterCols_LSX; } } diff --git a/source/scale_common.cc b/source/scale_common.cc index 6c4c6dac1..f3f130636 100644 --- a/source/scale_common.cc +++ b/source/scale_common.cc @@ -1437,11 +1437,19 @@ void ScalePlaneVertical(int src_height, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(dst_width_bytes, 32)) { + if (IS_ALIGNED(dst_width_bytes, 16)) { InterpolateRow = InterpolateRow_LSX; } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(dst_width_bytes, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; diff --git a/source/scale_lsx.cc b/source/scale_lsx.cc index bfe5e9fbb..e44596781 100644 --- a/source/scale_lsx.cc +++ b/source/scale_lsx.cc @@ -1,8 +1,6 @@ /* * Copyright 2022 The LibYuv Project Authors. All rights reserved. * - * Copyright (c) 2022 Loongson Technology Corporation Limited - * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found @@ -15,88 +13,121 @@ #include "libyuv/scale_row.h" #if !defined(LIBYUV_DISABLE_LSX) && defined(__loongarch_sx) -#include "libyuv/loongson_intrinsics.h" #ifdef __cplusplus namespace libyuv { extern "C" { #endif -#define LOAD_DATA(_src, _in, _out) \ - { \ - int _tmp1, _tmp2, _tmp3, _tmp4; \ - DUP4_ARG2(__lsx_vpickve2gr_w, _in, 0, _in, 1, _in, 2, _in, 3, _tmp1, \ - _tmp2, _tmp3, _tmp4); \ - _out = __lsx_vinsgr2vr_w(_out, _src[_tmp1], 0); \ - _out = __lsx_vinsgr2vr_w(_out, _src[_tmp2], 1); \ - _out = __lsx_vinsgr2vr_w(_out, _src[_tmp3], 2); \ - _out = __lsx_vinsgr2vr_w(_out, _src[_tmp4], 3); \ - } +static const uint64_t kScale38Shuf[2] = {0x13100e0b08060300ULL, + 0x000000001e1b1816ULL}; +static const uint64_t kScale38BoxShuf[2] = {0x0a08160604120200ULL, + 0x000000001e0e0c1aULL}; +static const uint64_t kScale34Shuf[2] = {0x0908070504030100ULL, + 0x000000000f0d0c0bULL}; +static const uint64_t kScale34BoxLo[2] = {0x0504030202010100ULL, + 0x0000000007060605ULL}; +static const uint64_t kScale34BoxHi[2] = {0x0d0c0b0a0a090908ULL, + 0x000000000f0e0e0dULL}; +static const uint64_t kScale34BoxWt[2] = {0x0103030102020103ULL, + 0x0000000003010202ULL}; +static const uint64_t kScale34Pack12[2] = {0x0908050403020100ULL, + 0x000000000d0c0b0aULL}; +static const uint64_t kRamp0123[2] = {0x0000000100000000ULL, + 0x0000000300000002ULL}; void ScaleARGBRowDown2_LSX(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { - int x; - int len = dst_width / 4; (void)src_stride; - __m128i src0, src1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - dst0 = __lsx_vpickod_w(src1, src0); - __lsx_vst(dst0, dst_argb, 0); - src_argb += 32; - dst_argb += 16; - } + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vld $vr1, %[src], 16 \n" + "vpickod.w $vr0, $vr1, $vr0 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$vr0", "$vr1"); } void ScaleARGBRowDown2Linear_LSX(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { - int x; - int len = dst_width / 4; (void)src_stride; - __m128i src0, src1, tmp0, tmp1, dst0; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_argb, 0, src_argb, 16, src0, src1); - tmp0 = __lsx_vpickev_w(src1, src0); - tmp1 = __lsx_vpickod_w(src1, src0); - dst0 = __lsx_vavgr_bu(tmp1, tmp0); - __lsx_vst(dst0, dst_argb, 0); - src_argb += 32; - dst_argb += 16; - } + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vld $vr1, %[src], 16 \n" + "vpickev.w $vr2, $vr1, $vr0 \n" + "vpickod.w $vr3, $vr1, $vr0 \n" + "vavgr.bu $vr0, $vr3, $vr2 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); } void ScaleARGBRowDown2Box_LSX(const uint8_t* src_argb, ptrdiff_t src_stride, uint8_t* dst_argb, int dst_width) { - int x; - int len = dst_width / 4; - const uint8_t* s = src_argb; const uint8_t* t = src_argb + src_stride; - __m128i src0, src1, src2, src3, tmp0, tmp1, tmp2, tmp3, dst0; - __m128i reg0, reg1, reg2, reg3; - __m128i shuff = {0x0703060205010400, 0x0F0B0E0A0D090C08}; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, s, 0, s, 16, src0, src1); - DUP2_ARG2(__lsx_vld, t, 0, t, 16, src2, src3); - DUP4_ARG3(__lsx_vshuf_b, src0, src0, shuff, src1, src1, shuff, src2, src2, - shuff, src3, src3, shuff, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG2(__lsx_vhaddw_hu_bu, tmp0, tmp0, tmp1, tmp1, tmp2, tmp2, tmp3, - tmp3, reg0, reg1, reg2, reg3); - DUP2_ARG2(__lsx_vsadd_hu, reg0, reg2, reg1, reg3, reg0, reg1); - dst0 = __lsx_vsrarni_b_h(reg1, reg0, 2); - __lsx_vst(dst0, dst_argb, 0); - s += 32; - t += 32; - dst_argb += 16; - } + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[s], 0 \n" + "vld $vr1, %[s], 16 \n" + "vld $vr2, %[t], 0 \n" + "vld $vr3, %[t], 16 \n" + "vpickev.w $vr4, $vr1, $vr0 \n" + "vpickod.w $vr5, $vr1, $vr0 \n" + "vpickev.w $vr6, $vr3, $vr2 \n" + "vpickod.w $vr7, $vr3, $vr2 \n" + "vaddwev.h.bu $vr0, $vr4, $vr5 \n" + "vaddwod.h.bu $vr1, $vr4, $vr5 \n" + "vaddwev.h.bu $vr2, $vr6, $vr7 \n" + "vaddwod.h.bu $vr3, $vr6, $vr7 \n" + "vadd.h $vr0, $vr0, $vr2 \n" + "vadd.h $vr1, $vr1, $vr3 \n" + "vsrari.h $vr0, $vr0, 2 \n" + "vsrari.h $vr1, $vr1, 2 \n" + "vpackev.b $vr0, $vr1, $vr0 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[s], %[s], 32 \n" + "addi.d %[t], %[t], 32 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [s] "+r"(src_argb), // %[s] + [t] "+r"(t), // %[t] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); } void ScaleARGBRowDownEven_LSX(const uint8_t* src_argb, @@ -104,27 +135,38 @@ void ScaleARGBRowDownEven_LSX(const uint8_t* src_argb, int32_t src_stepx, uint8_t* dst_argb, int dst_width) { - int x; - int len = dst_width / 4; - int32_t stepx = src_stepx << 2; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; + uint32_t r0, r1, r2, r3; (void)src_stride; - __m128i dst0, dst1, dst2, dst3; - - for (x = 0; x < len; x++) { - dst0 = __lsx_vldrepl_w(src_argb, 0); - src_argb += stepx; - dst1 = __lsx_vldrepl_w(src_argb, 0); - src_argb += stepx; - dst2 = __lsx_vldrepl_w(src_argb, 0); - src_argb += stepx; - dst3 = __lsx_vldrepl_w(src_argb, 0); - src_argb += stepx; - __lsx_vstelm_w(dst0, dst_argb, 0, 0); - __lsx_vstelm_w(dst1, dst_argb, 4, 0); - __lsx_vstelm_w(dst2, dst_argb, 8, 0); - __lsx_vstelm_w(dst3, dst_argb, 12, 0); - dst_argb += 16; - } + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "1: \n" + "ld.w %[r0], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r1], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r2], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "ld.w %[r3], %[src], 0 \n" + "add.d %[src], %[src], %[step] \n" + "st.w %[r0], %[dst], 0 \n" + "st.w %[r1], %[dst], 4 \n" + "st.w %[r2], %[dst], 8 \n" + "st.w %[r3], %[dst], 12 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_argb), // %[src] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width), // %[w] + [r0] "=&r"(r0), // %[r0] + [r1] "=&r"(r1), // %[r1] + [r2] "=&r"(r2), // %[r2] + [r3] "=&r"(r3) // %[r3] + : [step] "r"(step) // %[step] + : "memory"); } void ScaleARGBRowDownEvenBox_LSX(const uint8_t* src_argb, @@ -132,315 +174,411 @@ void ScaleARGBRowDownEvenBox_LSX(const uint8_t* src_argb, int src_stepx, uint8_t* dst_argb, int dst_width) { - int x; - int len = dst_width / 4; - int32_t stepx = src_stepx * 4; + const ptrdiff_t step = (ptrdiff_t)src_stepx * 4; const uint8_t* next_argb = src_argb + src_stride; - __m128i src0, src1, src2, src3; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128i reg0, reg1, dst0; + uint64_t r0, r1; + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "1: \n" + "ld.d %[r0], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "ld.d %[r1], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "vinsgr2vr.d $vr0, %[r0], 0 \n" + "vinsgr2vr.d $vr0, %[r1], 1 \n" + "ld.d %[r0], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "ld.d %[r1], %[s], 0 \n" + "add.d %[s], %[s], %[step] \n" + "vinsgr2vr.d $vr1, %[r0], 0 \n" + "vinsgr2vr.d $vr1, %[r1], 1 \n" - for (x = 0; x < len; x++) { - tmp0 = __lsx_vldrepl_d(src_argb, 0); - src_argb += stepx; - tmp1 = __lsx_vldrepl_d(src_argb, 0); - src_argb += stepx; - tmp2 = __lsx_vldrepl_d(src_argb, 0); - src_argb += stepx; - tmp3 = __lsx_vldrepl_d(src_argb, 0); - src_argb += stepx; - tmp4 = __lsx_vldrepl_d(next_argb, 0); - next_argb += stepx; - tmp5 = __lsx_vldrepl_d(next_argb, 0); - next_argb += stepx; - tmp6 = __lsx_vldrepl_d(next_argb, 0); - next_argb += stepx; - tmp7 = __lsx_vldrepl_d(next_argb, 0); - next_argb += stepx; - DUP4_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6, - src0, src1, src2, src3); - DUP2_ARG2(__lsx_vaddwev_h_bu, src0, src2, src1, src3, tmp0, tmp2); - DUP2_ARG2(__lsx_vaddwod_h_bu, src0, src2, src1, src3, tmp1, tmp3); - DUP2_ARG2(__lsx_vpackev_w, tmp1, tmp0, tmp3, tmp2, reg0, reg1); - DUP2_ARG2(__lsx_vpackod_w, tmp1, tmp0, tmp3, tmp2, tmp4, tmp5); - DUP2_ARG2(__lsx_vadd_h, reg0, tmp4, reg1, tmp5, reg0, reg1); - dst0 = __lsx_vsrarni_b_h(reg1, reg0, 2); - dst0 = __lsx_vshuf4i_b(dst0, 0xD8); - __lsx_vst(dst0, dst_argb, 0); - dst_argb += 16; - } + "ld.d %[r0], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "ld.d %[r1], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "vinsgr2vr.d $vr2, %[r0], 0 \n" + "vinsgr2vr.d $vr2, %[r1], 1 \n" + "ld.d %[r0], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "ld.d %[r1], %[t], 0 \n" + "add.d %[t], %[t], %[step] \n" + "vinsgr2vr.d $vr3, %[r0], 0 \n" + "vinsgr2vr.d $vr3, %[r1], 1 \n" + + "vpickev.w $vr4, $vr1, $vr0 \n" + "vpickod.w $vr5, $vr1, $vr0 \n" + "vpickev.w $vr6, $vr3, $vr2 \n" + "vpickod.w $vr7, $vr3, $vr2 \n" + "vaddwev.h.bu $vr0, $vr4, $vr5 \n" + "vaddwod.h.bu $vr1, $vr4, $vr5 \n" + "vaddwev.h.bu $vr2, $vr6, $vr7 \n" + "vaddwod.h.bu $vr3, $vr6, $vr7 \n" + "vadd.h $vr0, $vr0, $vr2 \n" + "vadd.h $vr1, $vr1, $vr3 \n" + "vsrari.h $vr0, $vr0, 2 \n" + "vsrari.h $vr1, $vr1, 2 \n" + "vpackev.b $vr0, $vr1, $vr0 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [s] "+r"(src_argb), // %[s] + [t] "+r"(next_argb), // %[t] + [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width), // %[w] + [r0] "=&r"(r0), // %[r0] + [r1] "=&r"(r1) // %[r1] + : [step] "r"(step) // %[step] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); } void ScaleRowDown2_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - int x; - int len = dst_width / 32; - __m128i src0, src1, src2, src3, dst0, dst1; (void)src_stride; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, - src0, src1, src2, src3); - DUP2_ARG2(__lsx_vpickod_b, src1, src0, src3, src2, dst0, dst1); - __lsx_vst(dst0, dst, 0); - __lsx_vst(dst1, dst, 16); - src_ptr += 64; - dst += 32; - } + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vld $vr1, %[src], 16 \n" + "vpickod.b $vr0, $vr1, $vr0 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$vr0", "$vr1"); } void ScaleRowDown2Linear_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - int x; - int len = dst_width / 32; - __m128i src0, src1, src2, src3; - __m128i tmp0, tmp1, tmp2, tmp3, dst0, dst1; (void)src_stride; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, - src0, src1, src2, src3); - DUP2_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, tmp0, tmp2); - DUP2_ARG2(__lsx_vpickod_b, src1, src0, src3, src2, tmp1, tmp3); - DUP2_ARG2(__lsx_vavgr_bu, tmp0, tmp1, tmp2, tmp3, dst0, dst1); - __lsx_vst(dst0, dst, 0); - __lsx_vst(dst1, dst, 16); - src_ptr += 64; - dst += 32; - } + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vld $vr1, %[src], 16 \n" + "vpickev.b $vr2, $vr1, $vr0 \n" + "vpickod.b $vr3, $vr1, $vr0 \n" + "vavgr.bu $vr0, $vr3, $vr2 \n" + "vst $vr0, %[dst], 0 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); } void ScaleRowDown2Box_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - int x; - int len = dst_width / 32; const uint8_t* src_nex = src_ptr + src_stride; - __m128i src0, src1, src2, src3, src4, src5, src6, src7; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128i dst0, dst1; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, - src0, src1, src2, src3); - DUP4_ARG2(__lsx_vld, src_nex, 0, src_nex, 16, src_nex, 32, src_nex, 48, - src4, src5, src6, src7); - DUP4_ARG2(__lsx_vaddwev_h_bu, src0, src4, src1, src5, src2, src6, src3, - src7, tmp0, tmp2, tmp4, tmp6); - DUP4_ARG2(__lsx_vaddwod_h_bu, src0, src4, src1, src5, src2, src6, src3, - src7, tmp1, tmp3, tmp5, tmp7); - DUP4_ARG2(__lsx_vadd_h, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, - tmp0, tmp1, tmp2, tmp3); - DUP2_ARG3(__lsx_vsrarni_b_h, tmp1, tmp0, 2, tmp3, tmp2, 2, dst0, dst1); - __lsx_vst(dst0, dst, 0); - __lsx_vst(dst1, dst, 16); - src_ptr += 64; - src_nex += 64; - dst += 32; - } + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vld $vr1, %[src], 16 \n" + "vld $vr2, %[nex], 0 \n" + "vld $vr3, %[nex], 16 \n" + "vaddwev.h.bu $vr4, $vr0, $vr2 \n" + "vaddwod.h.bu $vr5, $vr0, $vr2 \n" + "vaddwev.h.bu $vr6, $vr1, $vr3 \n" + "vaddwod.h.bu $vr7, $vr1, $vr3 \n" + "vadd.h $vr4, $vr4, $vr5 \n" + "vadd.h $vr6, $vr6, $vr7 \n" + "vsrarni.b.h $vr6, $vr4, 2 \n" + "vst $vr6, %[dst], 0 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[nex], %[nex], 32 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [nex] "+r"(src_nex), // %[nex] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); } void ScaleRowDown4_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - int x; - int len = dst_width / 16; - __m128i src0, src1, src2, src3, tmp0, tmp1, dst0; (void)src_stride; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, - src0, src1, src2, src3); - DUP2_ARG2(__lsx_vpickev_b, src1, src0, src3, src2, tmp0, tmp1); - dst0 = __lsx_vpickod_b(tmp1, tmp0); - __lsx_vst(dst0, dst, 0); - src_ptr += 64; - dst += 16; - } + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vld $vr1, %[src], 16 \n" + "vpickev.b $vr0, $vr1, $vr0 \n" + "vpickod.b $vr0, $vr0, $vr0 \n" + "vstelm.d $vr0, %[dst], 0, 0 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 8 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$vr0", "$vr1"); } void ScaleRowDown4Box_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - int x; - int len = dst_width / 16; const uint8_t* ptr1 = src_ptr + src_stride; const uint8_t* ptr2 = ptr1 + src_stride; const uint8_t* ptr3 = ptr2 + src_stride; - __m128i src0, src1, src2, src3, src4, src5, src6, src7; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7, dst0; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, - src0, src1, src2, src3); - DUP4_ARG2(__lsx_vld, ptr1, 0, ptr1, 16, ptr1, 32, ptr1, 48, src4, src5, - src6, src7); - DUP4_ARG2(__lsx_vaddwev_h_bu, src0, src4, src1, src5, src2, src6, src3, - src7, tmp0, tmp2, tmp4, tmp6); - DUP4_ARG2(__lsx_vaddwod_h_bu, src0, src4, src1, src5, src2, src6, src3, - src7, tmp1, tmp3, tmp5, tmp7); - DUP4_ARG2(__lsx_vadd_h, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, - reg0, reg1, reg2, reg3); - DUP4_ARG2(__lsx_vld, ptr2, 0, ptr2, 16, ptr2, 32, ptr2, 48, src0, src1, - src2, src3); - DUP4_ARG2(__lsx_vld, ptr3, 0, ptr3, 16, ptr3, 32, ptr3, 48, src4, src5, - src6, src7); - DUP4_ARG2(__lsx_vaddwev_h_bu, src0, src4, src1, src5, src2, src6, src3, - src7, tmp0, tmp2, tmp4, tmp6); - DUP4_ARG2(__lsx_vaddwod_h_bu, src0, src4, src1, src5, src2, src6, src3, - src7, tmp1, tmp3, tmp5, tmp7); - DUP4_ARG2(__lsx_vadd_h, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, - reg4, reg5, reg6, reg7); - DUP4_ARG2(__lsx_vadd_h, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, - reg0, reg1, reg2, reg3); - DUP4_ARG2(__lsx_vhaddw_wu_hu, reg0, reg0, reg1, reg1, reg2, reg2, reg3, - reg3, reg0, reg1, reg2, reg3); - DUP2_ARG3(__lsx_vsrarni_h_w, reg1, reg0, 4, reg3, reg2, 4, tmp0, tmp1); - dst0 = __lsx_vpickev_b(tmp1, tmp0); - __lsx_vst(dst0, dst, 0); - src_ptr += 64; - ptr1 += 64; - ptr2 += 64; - ptr3 += 64; - dst += 16; - } + asm volatile( + "srli.w %[w], %[w], 3 \n" + "beqz %[w], 2f \n" + "1: \n" + "vld $vr0, %[p0], 0 \n" + "vld $vr1, %[p0], 16 \n" + "vld $vr2, %[p1], 0 \n" + "vld $vr3, %[p1], 16 \n" + "vld $vr4, %[p2], 0 \n" + "vld $vr5, %[p2], 16 \n" + "vld $vr6, %[p3], 0 \n" + "vld $vr7, %[p3], 16 \n" + "vhaddw.hu.bu $vr0, $vr0, $vr0 \n" + "vhaddw.hu.bu $vr1, $vr1, $vr1 \n" + "vhaddw.hu.bu $vr2, $vr2, $vr2 \n" + "vhaddw.hu.bu $vr3, $vr3, $vr3 \n" + "vhaddw.hu.bu $vr4, $vr4, $vr4 \n" + "vhaddw.hu.bu $vr5, $vr5, $vr5 \n" + "vhaddw.hu.bu $vr6, $vr6, $vr6 \n" + "vhaddw.hu.bu $vr7, $vr7, $vr7 \n" + "vadd.h $vr0, $vr0, $vr2 \n" + "vadd.h $vr1, $vr1, $vr3 \n" + "vadd.h $vr4, $vr4, $vr6 \n" + "vadd.h $vr5, $vr5, $vr7 \n" + "vadd.h $vr0, $vr0, $vr4 \n" + "vadd.h $vr1, $vr1, $vr5 \n" + "vhaddw.wu.hu $vr0, $vr0, $vr0 \n" + "vhaddw.wu.hu $vr1, $vr1, $vr1 \n" + "vsrarni.h.w $vr1, $vr0, 4 \n" + "vsrlni.b.h $vr1, $vr1, 0 \n" + "vstelm.d $vr1, %[dst], 0, 0 \n" + "addi.d %[p0], %[p0], 32 \n" + "addi.d %[p1], %[p1], 32 \n" + "addi.d %[p2], %[p2], 32 \n" + "addi.d %[p3], %[p3], 32 \n" + "addi.d %[dst], %[dst], 8 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [p0] "+r"(src_ptr), // %[p0] + [p1] "+r"(ptr1), // %[p1] + [p2] "+r"(ptr2), // %[p2] + [p3] "+r"(ptr3), // %[p3] + [dst] "+r"(dst), // %[dst] + [w] "+r"(dst_width) // %[w] + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); } void ScaleRowDown38_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - int x, len; - __m128i src0, src1, tmp0; - __m128i shuff = {0x13100E0B08060300, 0x000000001E1B1816}; - - assert(dst_width % 3 == 0); - len = dst_width / 12; + int count = dst_width / 12; (void)src_stride; - - for (x = 0; x < len; x++) { - DUP2_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src0, src1); - tmp0 = __lsx_vshuf_b(src1, src0, shuff); - __lsx_vstelm_d(tmp0, dst, 0, 0); - __lsx_vstelm_w(tmp0, dst, 8, 2); - src_ptr += 32; - dst += 12; - } + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr2, %[shuf], 0 \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vld $vr1, %[src], 16 \n" + "vshuf.b $vr0, $vr1, $vr0, $vr2 \n" + "vstelm.d $vr0, %[dst], 0, 0 \n" + "vstelm.w $vr0, %[dst], 8, 2 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [shuf] "r"(kScale38Shuf) // %[shuf] + : "memory", "$vr0", "$vr1", "$vr2"); } void ScaleRowDown38_2_Box_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - int x, len; const uint8_t* src_nex = src_ptr + src_stride; - __m128i src0, src1, src2, src3, dst0; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128i reg0, reg1, reg2, reg3; - __m128i shuff = {0x0A08160604120200, 0x000000001E0E0C1A}; - __m128i const_0x2AAA = __lsx_vreplgr2vr_h(0x2AAA); - __m128i const_0x4000 = __lsx_vreplgr2vr_w(0x4000); - - assert((dst_width % 3 == 0) && (dst_width > 0)); - len = dst_width / 12; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_nex, 0, src_nex, 16, src0, - src1, src2, src3); - DUP2_ARG2(__lsx_vaddwev_h_bu, src0, src2, src1, src3, tmp0, tmp2); - DUP2_ARG2(__lsx_vaddwod_h_bu, src0, src2, src1, src3, tmp1, tmp3); - DUP2_ARG2(__lsx_vpickev_h, tmp2, tmp0, tmp3, tmp1, reg0, reg1); - DUP2_ARG2(__lsx_vpackod_h, tmp1, tmp0, tmp3, tmp2, reg2, reg3); - tmp4 = __lsx_vpickev_w(reg3, reg2); - tmp5 = __lsx_vadd_h(reg0, reg1); - tmp6 = __lsx_vadd_h(tmp5, tmp4); - tmp7 = __lsx_vmuh_h(tmp6, const_0x2AAA); - tmp0 = __lsx_vpickod_w(reg3, reg2); - tmp1 = __lsx_vhaddw_wu_hu(tmp0, tmp0); - tmp2 = __lsx_vmul_w(tmp1, const_0x4000); - dst0 = __lsx_vshuf_b(tmp2, tmp7, shuff); - __lsx_vstelm_d(dst0, dst_ptr, 0, 0); - __lsx_vstelm_w(dst0, dst_ptr, 8, 2); - src_ptr += 32; - src_nex += 32; - dst_ptr += 12; - } + int count = dst_width / 12; + int c2aaa = 0x2aaa; + int c4000 = 0x4000; + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr8, %[shuf], 0 \n" + "vreplgr2vr.h $vr9, %[c2aaa] \n" + "vreplgr2vr.w $vr10, %[c4000] \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vld $vr1, %[src], 16 \n" + "vld $vr2, %[nex], 0 \n" + "vld $vr3, %[nex], 16 \n" + "vaddwev.h.bu $vr4, $vr0, $vr2 \n" + "vaddwod.h.bu $vr5, $vr0, $vr2 \n" + "vaddwev.h.bu $vr6, $vr1, $vr3 \n" + "vaddwod.h.bu $vr7, $vr1, $vr3 \n" + "vpickev.h $vr0, $vr6, $vr4 \n" + "vpickev.h $vr1, $vr7, $vr5 \n" + "vpackod.h $vr2, $vr5, $vr4 \n" + "vpackod.h $vr3, $vr7, $vr6 \n" + "vpickev.w $vr4, $vr3, $vr2 \n" + "vadd.h $vr0, $vr0, $vr1 \n" + "vadd.h $vr0, $vr0, $vr4 \n" + "vmuh.h $vr0, $vr0, $vr9 \n" + "vpickod.w $vr2, $vr3, $vr2 \n" + "vhaddw.wu.hu $vr2, $vr2, $vr2 \n" + "vmul.w $vr2, $vr2, $vr10 \n" + "vshuf.b $vr0, $vr2, $vr0, $vr8 \n" + "vstelm.d $vr0, %[dst], 0, 0 \n" + "vstelm.w $vr0, %[dst], 8, 2 \n" + "addi.d %[src], %[src], 32 \n" + "addi.d %[nex], %[nex], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [nex] "+r"(src_nex), // %[nex] + [dst] "+r"(dst_ptr), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [shuf] "r"(kScale38BoxShuf), // %[shuf] + [c2aaa] "r"(c2aaa), // %[c2aaa] + [c4000] "r"(c4000) // %[c4000] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10"); } void ScaleRowDown38_3_Box_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst_ptr, int dst_width) { - int x, len; const uint8_t* ptr1 = src_ptr + src_stride; const uint8_t* ptr2 = ptr1 + src_stride; - __m128i src0, src1, src2, src3, src4, src5; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128i reg0, reg1, reg2, reg3, dst0; - __m128i zero = __lsx_vldi(0); - __m128i shuff = {0x0A08160604120200, 0x000000001E0E0C1A}; - __m128i const_0x1C71 = __lsx_vreplgr2vr_h(0x1C71); - __m128i const_0x2AAA = __lsx_vreplgr2vr_w(0x2AAA); - - assert((dst_width % 3 == 0) && (dst_width > 0)); - len = dst_width / 12; - - for (x = 0; x < len; x++) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, ptr1, 0, ptr1, 16, src0, src1, - src2, src3); - DUP2_ARG2(__lsx_vld, ptr2, 0, ptr2, 16, src4, src5); - DUP2_ARG2(__lsx_vaddwev_h_bu, src0, src2, src1, src3, tmp0, tmp2); - DUP2_ARG2(__lsx_vaddwod_h_bu, src0, src2, src1, src3, tmp1, tmp3); - DUP2_ARG2(__lsx_vpackev_b, zero, src4, zero, src5, tmp4, tmp6); - DUP2_ARG2(__lsx_vpackod_b, zero, src4, zero, src5, tmp5, tmp7); - DUP4_ARG2(__lsx_vadd_h, tmp0, tmp4, tmp1, tmp5, tmp2, tmp6, tmp3, tmp7, - tmp0, tmp1, tmp2, tmp3); - DUP2_ARG2(__lsx_vpickev_h, tmp2, tmp0, tmp3, tmp1, reg0, reg1); - DUP2_ARG2(__lsx_vpackod_h, tmp1, tmp0, tmp3, tmp2, reg2, reg3); - tmp4 = __lsx_vpickev_w(reg3, reg2); - tmp5 = __lsx_vadd_h(reg0, reg1); - tmp6 = __lsx_vadd_h(tmp5, tmp4); - tmp7 = __lsx_vmuh_h(tmp6, const_0x1C71); - tmp0 = __lsx_vpickod_w(reg3, reg2); - tmp1 = __lsx_vhaddw_wu_hu(tmp0, tmp0); - tmp2 = __lsx_vmul_w(tmp1, const_0x2AAA); - dst0 = __lsx_vshuf_b(tmp2, tmp7, shuff); - __lsx_vstelm_d(dst0, dst_ptr, 0, 0); - __lsx_vstelm_w(dst0, dst_ptr, 8, 2); - src_ptr += 32; - ptr1 += 32; - ptr2 += 32; - dst_ptr += 12; - } + int count = dst_width / 12; + int c1c71 = 0x1c71; + int c2aaa = 0x2aaa; + asm volatile( + "beqz %[cnt], 2f \n" + "vldi $vr11, 0 \n" + "vld $vr8, %[shuf], 0 \n" + "vreplgr2vr.h $vr9, %[c1c71] \n" + "vreplgr2vr.w $vr10, %[c2aaa] \n" + "1: \n" + "vld $vr0, %[p0], 0 \n" + "vld $vr1, %[p0], 16 \n" + "vld $vr2, %[p1], 0 \n" + "vld $vr3, %[p1], 16 \n" + "vaddwev.h.bu $vr4, $vr0, $vr2 \n" + "vaddwod.h.bu $vr5, $vr0, $vr2 \n" + "vaddwev.h.bu $vr6, $vr1, $vr3 \n" + "vaddwod.h.bu $vr7, $vr1, $vr3 \n" + "vld $vr0, %[p2], 0 \n" + "vld $vr1, %[p2], 16 \n" + "vpackev.b $vr2, $vr11, $vr0 \n" + "vpackod.b $vr3, $vr11, $vr0 \n" + "vadd.h $vr4, $vr4, $vr2 \n" + "vadd.h $vr5, $vr5, $vr3 \n" + "vpackev.b $vr2, $vr11, $vr1 \n" + "vpackod.b $vr3, $vr11, $vr1 \n" + "vadd.h $vr6, $vr6, $vr2 \n" + "vadd.h $vr7, $vr7, $vr3 \n" + "vpickev.h $vr0, $vr6, $vr4 \n" + "vpickev.h $vr1, $vr7, $vr5 \n" + "vpackod.h $vr2, $vr5, $vr4 \n" + "vpackod.h $vr3, $vr7, $vr6 \n" + "vpickev.w $vr4, $vr3, $vr2 \n" + "vadd.h $vr0, $vr0, $vr1 \n" + "vadd.h $vr0, $vr0, $vr4 \n" + "vmuh.h $vr0, $vr0, $vr9 \n" + "vpickod.w $vr2, $vr3, $vr2 \n" + "vhaddw.wu.hu $vr2, $vr2, $vr2 \n" + "vmul.w $vr2, $vr2, $vr10 \n" + "vshuf.b $vr0, $vr2, $vr0, $vr8 \n" + "vstelm.d $vr0, %[dst], 0, 0 \n" + "vstelm.w $vr0, %[dst], 8, 2 \n" + "addi.d %[p0], %[p0], 32 \n" + "addi.d %[p1], %[p1], 32 \n" + "addi.d %[p2], %[p2], 32 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [p0] "+r"(src_ptr), // %[p0] + [p1] "+r"(ptr1), // %[p1] + [p2] "+r"(ptr2), // %[p2] + [dst] "+r"(dst_ptr), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [shuf] "r"(kScale38BoxShuf), // %[shuf] + [c1c71] "r"(c1c71), // %[c1c71] + [c2aaa] "r"(c2aaa) // %[c2aaa] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9", "$vr10", "$vr11"); } void ScaleAddRow_LSX(const uint8_t* src_ptr, uint16_t* dst_ptr, int src_width) { - int x; - int len = src_width / 16; - __m128i src0, tmp0, tmp1, dst0, dst1; - __m128i zero = __lsx_vldi(0); - - assert(src_width > 0); - - for (x = 0; x < len; x++) { - src0 = __lsx_vld(src_ptr, 0); - DUP2_ARG2(__lsx_vld, dst_ptr, 0, dst_ptr, 16, dst0, dst1); - tmp0 = __lsx_vilvl_b(zero, src0); - tmp1 = __lsx_vilvh_b(zero, src0); - DUP2_ARG2(__lsx_vadd_h, dst0, tmp0, dst1, tmp1, dst0, dst1); - __lsx_vst(dst0, dst_ptr, 0); - __lsx_vst(dst1, dst_ptr, 16); - src_ptr += 16; - dst_ptr += 16; - } + asm volatile( + "srli.w %[w], %[w], 4 \n" + "beqz %[w], 2f \n" + "vldi $vr0, 0 \n" + "1: \n" + "vld $vr1, %[src], 0 \n" + "vld $vr2, %[dst], 0 \n" + "vld $vr3, %[dst], 16 \n" + "vilvl.b $vr4, $vr0, $vr1 \n" + "vilvh.b $vr5, $vr0, $vr1 \n" + "vadd.h $vr2, $vr2, $vr4 \n" + "vadd.h $vr3, $vr3, $vr5 \n" + "vst $vr2, %[dst], 0 \n" + "vst $vr3, %[dst], 16 \n" + "addi.d %[src], %[src], 16 \n" + "addi.d %[dst], %[dst], 32 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst_ptr), // %[dst] + [w] "+r"(src_width) // %[w] + : + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5"); } void ScaleFilterCols_LSX(uint8_t* dst_ptr, @@ -448,61 +586,64 @@ void ScaleFilterCols_LSX(uint8_t* dst_ptr, int dst_width, int x, int dx) { - int j; - int len = dst_width / 16; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; - __m128i vec0, vec1, dst0; - __m128i vec_x = __lsx_vreplgr2vr_w(x); - __m128i vec_dx = __lsx_vreplgr2vr_w(dx); - __m128i const1 = __lsx_vreplgr2vr_w(0xFFFF); - __m128i const2 = __lsx_vreplgr2vr_w(0x40); - __m128i const_tmp = {0x0000000100000000, 0x0000000300000002}; - - vec0 = __lsx_vmul_w(vec_dx, const_tmp); - vec1 = __lsx_vslli_w(vec_dx, 2); - vec_x = __lsx_vadd_w(vec_x, vec0); - - for (j = 0; j < len; j++) { - tmp0 = __lsx_vsrai_w(vec_x, 16); - tmp4 = __lsx_vand_v(vec_x, const1); - vec_x = __lsx_vadd_w(vec_x, vec1); - tmp1 = __lsx_vsrai_w(vec_x, 16); - tmp5 = __lsx_vand_v(vec_x, const1); - vec_x = __lsx_vadd_w(vec_x, vec1); - tmp2 = __lsx_vsrai_w(vec_x, 16); - tmp6 = __lsx_vand_v(vec_x, const1); - vec_x = __lsx_vadd_w(vec_x, vec1); - tmp3 = __lsx_vsrai_w(vec_x, 16); - tmp7 = __lsx_vand_v(vec_x, const1); - vec_x = __lsx_vadd_w(vec_x, vec1); - DUP4_ARG2(__lsx_vsrai_w, tmp4, 9, tmp5, 9, tmp6, 9, tmp7, 9, tmp4, tmp5, - tmp6, tmp7); - LOAD_DATA(src_ptr, tmp0, reg0); - LOAD_DATA(src_ptr, tmp1, reg1); - LOAD_DATA(src_ptr, tmp2, reg2); - LOAD_DATA(src_ptr, tmp3, reg3); - DUP4_ARG2(__lsx_vaddi_wu, tmp0, 1, tmp1, 1, tmp2, 1, tmp3, 1, tmp0, tmp1, - tmp2, tmp3); - LOAD_DATA(src_ptr, tmp0, reg4); - LOAD_DATA(src_ptr, tmp1, reg5); - LOAD_DATA(src_ptr, tmp2, reg6); - LOAD_DATA(src_ptr, tmp3, reg7); - DUP4_ARG2(__lsx_vsub_w, reg4, reg0, reg5, reg1, reg6, reg2, reg7, reg3, - reg4, reg5, reg6, reg7); - DUP4_ARG2(__lsx_vmul_w, reg4, tmp4, reg5, tmp5, reg6, tmp6, reg7, tmp7, - reg4, reg5, reg6, reg7); - DUP4_ARG2(__lsx_vadd_w, reg4, const2, reg5, const2, reg6, const2, reg7, - const2, reg4, reg5, reg6, reg7); - DUP4_ARG2(__lsx_vsrai_w, reg4, 7, reg5, 7, reg6, 7, reg7, 7, reg4, reg5, - reg6, reg7); - DUP4_ARG2(__lsx_vadd_w, reg0, reg4, reg1, reg5, reg2, reg6, reg3, reg7, - reg0, reg1, reg2, reg3); - DUP2_ARG2(__lsx_vpickev_h, reg1, reg0, reg3, reg2, tmp0, tmp1); - dst0 = __lsx_vpickev_b(tmp1, tmp0); - __lsx_vst(dst0, dst_ptr, 0); - dst_ptr += 16; - } + ptrdiff_t i0, i1, i2, i3; + uint32_t p0, p1; + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "vld $vr0, %[ramp], 0 \n" + "vreplgr2vr.w $vr1, %[dx] \n" + "vreplgr2vr.w $vr2, %[x] \n" + "vmul.w $vr0, $vr1, $vr0 \n" + "vslli.w $vr1, $vr1, 2 \n" + "vadd.w $vr2, $vr2, $vr0 \n" + "vldi $vr3, 0x7f \n" + "vsrli.w $vr3, $vr3, 24 \n" + "vaddi.wu $vr4, $vr3, 1 \n" + "1: \n" + "vsrai.w $vr5, $vr2, 16 \n" + "vsrai.w $vr6, $vr2, 9 \n" + "vand.v $vr6, $vr6, $vr3 \n" + "vsub.w $vr7, $vr4, $vr6 \n" + "vslli.w $vr6, $vr6, 8 \n" + "vor.v $vr6, $vr6, $vr7 \n" + "vadd.w $vr2, $vr2, $vr1 \n" + "vpickve2gr.w %[i0], $vr5, 0 \n" + "vpickve2gr.w %[i1], $vr5, 1 \n" + "vpickve2gr.w %[i2], $vr5, 2 \n" + "vpickve2gr.w %[i3], $vr5, 3 \n" + "ldx.h %[p0], %[src], %[i0] \n" + "ldx.h %[p1], %[src], %[i1] \n" + "vinsgr2vr.h $vr5, %[p0], 0 \n" + "vinsgr2vr.h $vr5, %[p1], 2 \n" + "ldx.h %[p0], %[src], %[i2] \n" + "ldx.h %[p1], %[src], %[i3] \n" + "vinsgr2vr.h $vr5, %[p0], 4 \n" + "vinsgr2vr.h $vr5, %[p1], 6 \n" + "vmulwev.h.bu $vr7, $vr5, $vr6 \n" + "vmaddwod.h.bu $vr7, $vr5, $vr6 \n" + "vsrari.h $vr5, $vr7, 7 \n" + "vpickev.b $vr5, $vr5, $vr5 \n" + "vpickev.b $vr5, $vr5, $vr5 \n" + "vstelm.w $vr5, %[dst], 0, 0 \n" + "addi.d %[dst], %[dst], 4 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [dst] "+r"(dst_ptr), // %[dst] + [w] "+r"(dst_width), // %[w] + [i0] "=&r"(i0), // %[i0] + [i1] "=&r"(i1), // %[i1] + [i2] "=&r"(i2), // %[i2] + [i3] "=&r"(i3), // %[i3] + [p0] "=&r"(p0), // %[p0] + [p1] "=&r"(p1) // %[p1] + : [src] "r"(src_ptr), // %[src] + [x] "r"(x), // %[x] + [dx] "r"(dx), // %[dx] + [ramp] "r"(kRamp0123) // %[ramp] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7"); } void ScaleARGBCols_LSX(uint8_t* dst_argb, @@ -510,26 +651,50 @@ void ScaleARGBCols_LSX(uint8_t* dst_argb, int dst_width, int x, int dx) { - const uint32_t* src = (const uint32_t*)src_argb; - uint32_t* dst = (uint32_t*)dst_argb; - int j; - int len = dst_width / 4; - __m128i tmp0, tmp1, tmp2, dst0; - __m128i vec_x = __lsx_vreplgr2vr_w(x); - __m128i vec_dx = __lsx_vreplgr2vr_w(dx); - __m128i const_tmp = {0x0000000100000000, 0x0000000300000002}; - - tmp0 = __lsx_vmul_w(vec_dx, const_tmp); - tmp1 = __lsx_vslli_w(vec_dx, 2); - vec_x = __lsx_vadd_w(vec_x, tmp0); - - for (j = 0; j < len; j++) { - tmp2 = __lsx_vsrai_w(vec_x, 16); - vec_x = __lsx_vadd_w(vec_x, tmp1); - LOAD_DATA(src, tmp2, dst0); - __lsx_vst(dst0, dst, 0); - dst += 4; - } + ptrdiff_t i0, i1, i2, i3; + uint32_t p0, p1; + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "vld $vr0, %[ramp], 0 \n" + "vreplgr2vr.w $vr1, %[dx] \n" + "vreplgr2vr.w $vr2, %[x] \n" + "vmul.w $vr0, $vr1, $vr0 \n" + "vslli.w $vr1, $vr1, 2 \n" + "vadd.w $vr2, $vr2, $vr0 \n" + "1: \n" + "vsrai.w $vr3, $vr2, 16 \n" + "vslli.w $vr3, $vr3, 2 \n" + "vadd.w $vr2, $vr2, $vr1 \n" + "vpickve2gr.w %[i0], $vr3, 0 \n" + "vpickve2gr.w %[i1], $vr3, 1 \n" + "vpickve2gr.w %[i2], $vr3, 2 \n" + "vpickve2gr.w %[i3], $vr3, 3 \n" + "ldx.w %[p0], %[src], %[i0] \n" + "ldx.w %[p1], %[src], %[i1] \n" + "st.w %[p0], %[dst], 0 \n" + "st.w %[p1], %[dst], 4 \n" + "ldx.w %[p0], %[src], %[i2] \n" + "ldx.w %[p1], %[src], %[i3] \n" + "st.w %[p0], %[dst], 8 \n" + "st.w %[p1], %[dst], 12 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width), // %[w] + [i0] "=&r"(i0), // %[i0] + [i1] "=&r"(i1), // %[i1] + [i2] "=&r"(i2), // %[i2] + [i3] "=&r"(i3), // %[i3] + [p0] "=&r"(p0), // %[p0] + [p1] "=&r"(p1) // %[p1] + : [src] "r"(src_argb), // %[src] + [x] "r"(x), // %[x] + [dx] "r"(dx), // %[dx] + [ramp] "r"(kRamp0123) // %[ramp] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3"); } void ScaleARGBFilterCols_LSX(uint8_t* dst_argb, @@ -537,76 +702,94 @@ void ScaleARGBFilterCols_LSX(uint8_t* dst_argb, int dst_width, int x, int dx) { - const uint32_t* src = (const uint32_t*)src_argb; - int j; - int len = dst_width / 8; - __m128i src0, src1, src2, src3; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; - __m128i reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7; - __m128i vec0, vec1, dst0, dst1; - __m128i vec_x = __lsx_vreplgr2vr_w(x); - __m128i vec_dx = __lsx_vreplgr2vr_w(dx); - __m128i const_tmp = {0x0000000100000000, 0x0000000300000002}; - __m128i const_7f = __lsx_vldi(0x7F); - - vec0 = __lsx_vmul_w(vec_dx, const_tmp); - vec1 = __lsx_vslli_w(vec_dx, 2); - vec_x = __lsx_vadd_w(vec_x, vec0); - - for (j = 0; j < len; j++) { - tmp0 = __lsx_vsrai_w(vec_x, 16); - reg0 = __lsx_vsrai_w(vec_x, 9); - vec_x = __lsx_vadd_w(vec_x, vec1); - tmp1 = __lsx_vsrai_w(vec_x, 16); - reg1 = __lsx_vsrai_w(vec_x, 9); - vec_x = __lsx_vadd_w(vec_x, vec1); - DUP2_ARG2(__lsx_vand_v, reg0, const_7f, reg1, const_7f, reg0, reg1); - DUP2_ARG2(__lsx_vshuf4i_b, reg0, 0, reg1, 0, reg0, reg1); - DUP2_ARG2(__lsx_vxor_v, reg0, const_7f, reg1, const_7f, reg2, reg3); - DUP2_ARG2(__lsx_vilvl_b, reg0, reg2, reg1, reg3, reg4, reg6); - DUP2_ARG2(__lsx_vilvh_b, reg0, reg2, reg1, reg3, reg5, reg7); - LOAD_DATA(src, tmp0, src0); - LOAD_DATA(src, tmp1, src1); - DUP2_ARG2(__lsx_vaddi_wu, tmp0, 1, tmp1, 1, tmp0, tmp1); - LOAD_DATA(src, tmp0, src2); - LOAD_DATA(src, tmp1, src3); - DUP2_ARG2(__lsx_vilvl_b, src2, src0, src3, src1, tmp4, tmp6); - DUP2_ARG2(__lsx_vilvh_b, src2, src0, src3, src1, tmp5, tmp7); - DUP4_ARG2(__lsx_vdp2_h_bu, tmp4, reg4, tmp5, reg5, tmp6, reg6, tmp7, reg7, - tmp0, tmp1, tmp2, tmp3); - DUP2_ARG3(__lsx_vsrani_b_h, tmp1, tmp0, 7, tmp3, tmp2, 7, dst0, dst1); - __lsx_vst(dst0, dst_argb, 0); - __lsx_vst(dst1, dst_argb, 16); - dst_argb += 32; - } + ptrdiff_t i0, i1, i2, i3; + uint64_t p0, p1; + asm volatile( + "srli.w %[w], %[w], 2 \n" + "beqz %[w], 2f \n" + "vld $vr0, %[ramp], 0 \n" + "vreplgr2vr.w $vr1, %[dx] \n" + "vreplgr2vr.w $vr2, %[x] \n" + "vmul.w $vr0, $vr1, $vr0 \n" + "vslli.w $vr1, $vr1, 2 \n" + "vadd.w $vr2, $vr2, $vr0 \n" + "vldi $vr3, 0x7f \n" + "1: \n" + "vsrai.w $vr4, $vr2, 16 \n" + "vslli.w $vr4, $vr4, 2 \n" + "vsrai.w $vr5, $vr2, 9 \n" + "vadd.w $vr2, $vr2, $vr1 \n" + "vand.v $vr5, $vr5, $vr3 \n" + "vshuf4i.b $vr5, $vr5, 0x00 \n" + "vxor.v $vr6, $vr5, $vr3 \n" + "vilvl.b $vr7, $vr5, $vr6 \n" + "vilvh.b $vr8, $vr5, $vr6 \n" + "vpickve2gr.w %[i0], $vr4, 0 \n" + "vpickve2gr.w %[i1], $vr4, 1 \n" + "vpickve2gr.w %[i2], $vr4, 2 \n" + "vpickve2gr.w %[i3], $vr4, 3 \n" + "ldx.d %[p0], %[src], %[i0] \n" + "ldx.d %[p1], %[src], %[i1] \n" + "vinsgr2vr.d $vr4, %[p0], 0 \n" + "vinsgr2vr.d $vr4, %[p1], 1 \n" + "ldx.d %[p0], %[src], %[i2] \n" + "ldx.d %[p1], %[src], %[i3] \n" + "vinsgr2vr.d $vr5, %[p0], 0 \n" + "vinsgr2vr.d $vr5, %[p1], 1 \n" + "vpickev.w $vr6, $vr5, $vr4 \n" + "vpickod.w $vr4, $vr5, $vr4 \n" + "vilvl.b $vr5, $vr4, $vr6 \n" + "vilvh.b $vr4, $vr4, $vr6 \n" + "vmulwev.h.bu $vr6, $vr5, $vr7 \n" + "vmaddwod.h.bu $vr6, $vr5, $vr7 \n" + "vmulwev.h.bu $vr7, $vr4, $vr8 \n" + "vmaddwod.h.bu $vr7, $vr4, $vr8 \n" + "vsrani.b.h $vr7, $vr6, 7 \n" + "vst $vr7, %[dst], 0 \n" + "addi.d %[dst], %[dst], 16 \n" + "addi.w %[w], %[w], -1 \n" + "bnez %[w], 1b \n" + "2: \n" + : [dst] "+r"(dst_argb), // %[dst] + [w] "+r"(dst_width), // %[w] + [i0] "=&r"(i0), // %[i0] + [i1] "=&r"(i1), // %[i1] + [i2] "=&r"(i2), // %[i2] + [i3] "=&r"(i3), // %[i3] + [p0] "=&r"(p0), // %[p0] + [p1] "=&r"(p1) // %[p1] + : [src] "r"(src_argb), // %[src] + [x] "r"(x), // %[x] + [dx] "r"(dx), // %[dx] + [ramp] "r"(kRamp0123) // %[ramp] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8"); } void ScaleRowDown34_LSX(const uint8_t* src_ptr, ptrdiff_t src_stride, uint8_t* dst, int dst_width) { - int x; + int count = dst_width / 12; (void)src_stride; - __m128i src0, src1, src2, src3; - __m128i dst0, dst1, dst2; - __m128i shuff0 = {0x0908070504030100, 0x141311100F0D0C0B}; - __m128i shuff1 = {0x0F0D0C0B09080705, 0x1918171514131110}; - __m128i shuff2 = {0x141311100F0D0C0B, 0x1F1D1C1B19181715}; - - assert((dst_width % 3 == 0) && (dst_width > 0)); - - for (x = 0; x < dst_width; x += 48) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, - src0, src1, src2, src3); - DUP2_ARG3(__lsx_vshuf_b, src1, src0, shuff0, src2, src1, shuff1, dst0, - dst1); - dst2 = __lsx_vshuf_b(src3, src2, shuff2); - __lsx_vst(dst0, dst, 0); - __lsx_vst(dst1, dst, 16); - __lsx_vst(dst2, dst, 32); - src_ptr += 64; - dst += 48; - } + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr1, %[shuf], 0 \n" + "1: \n" + "vld $vr0, %[src], 0 \n" + "vshuf.b $vr0, $vr0, $vr0, $vr1 \n" + "vstelm.d $vr0, %[dst], 0, 0 \n" + "vstelm.w $vr0, %[dst], 8, 2 \n" + "addi.d %[src], %[src], 16 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [dst] "+r"(dst), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [shuf] "r"(kScale34Shuf) // %[shuf] + : "memory", "$vr0", "$vr1"); } void ScaleRowDown34_0_Box_LSX(const uint8_t* src_ptr, @@ -614,63 +797,58 @@ void ScaleRowDown34_0_Box_LSX(const uint8_t* src_ptr, uint8_t* d, int dst_width) { const uint8_t* src_nex = src_ptr + src_stride; - int x; - __m128i src0, src1, src2, src3, src4, src5, src6, src7; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, tmp9; - __m128i tmp10, tmp11, dst0, dst1, dst2; - __m128i const0 = {0x0103030101010103, 0x0101010303010101}; - __m128i const1 = {0x0301010101030301, 0x0103030101010103}; - __m128i const2 = {0x0101010303010101, 0x0301010101030301}; - __m128i shuff0 = {0x0504030202010100, 0x0A09090807060605}; - __m128i shuff1 = {0x0F0E0E0D0D0C0B0A, 0x1514131212111110}; - __m128i shuff2 = {0x0A09090807060605, 0x0F0E0E0D0D0C0B0A}; - __m128i shift0 = {0x0002000200010002, 0x0001000200020001}; - __m128i shift1 = {0x0002000100020002, 0x0002000200010002}; - __m128i shift2 = {0x0001000200020001, 0x0002000100020002}; - - assert((dst_width % 3 == 0) && (dst_width > 0)); - - for (x = 0; x < dst_width; x += 48) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, - src0, src1, src2, src3); - DUP4_ARG2(__lsx_vld, src_nex, 0, src_nex, 16, src_nex, 32, src_nex, 48, - src4, src5, src6, src7); - DUP4_ARG3(__lsx_vshuf_b, src0, src0, shuff0, src1, src0, shuff1, src1, src1, - shuff2, src2, src2, shuff0, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG3(__lsx_vshuf_b, src3, src2, shuff1, src3, src3, shuff2, src4, src4, - shuff0, src5, src4, shuff1, tmp4, tmp5, tmp6, tmp7); - DUP4_ARG3(__lsx_vshuf_b, src5, src5, shuff2, src6, src6, shuff0, src7, src6, - shuff1, src7, src7, shuff2, tmp8, tmp9, tmp10, tmp11); - DUP4_ARG2(__lsx_vdp2_h_bu, tmp0, const0, tmp1, const1, tmp2, const2, tmp3, - const0, src0, src1, src2, src3); - DUP4_ARG2(__lsx_vdp2_h_bu, tmp4, const1, tmp5, const2, tmp6, const0, tmp7, - const1, src4, src5, src6, src7); - DUP4_ARG2(__lsx_vdp2_h_bu, tmp8, const2, tmp9, const0, tmp10, const1, tmp11, - const2, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG2(__lsx_vsrar_h, src0, shift0, src1, shift1, src2, shift2, src3, - shift0, src0, src1, src2, src3); - DUP4_ARG2(__lsx_vsrar_h, src4, shift1, src5, shift2, src6, shift0, src7, - shift1, src4, src5, src6, src7); - DUP4_ARG2(__lsx_vsrar_h, tmp0, shift2, tmp1, shift0, tmp2, shift1, tmp3, - shift2, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG2(__lsx_vslli_h, src0, 1, src1, 1, src2, 1, src3, 1, tmp5, tmp6, - tmp7, tmp8); - DUP2_ARG2(__lsx_vslli_h, src4, 1, src5, 1, tmp9, tmp10); - DUP4_ARG2(__lsx_vadd_h, src0, tmp5, src1, tmp6, src2, tmp7, src3, tmp8, - src0, src1, src2, src3); - DUP2_ARG2(__lsx_vadd_h, src4, tmp9, src5, tmp10, src4, src5); - DUP4_ARG2(__lsx_vadd_h, src0, src6, src1, src7, src2, tmp0, src3, tmp1, - src0, src1, src2, src3); - DUP2_ARG2(__lsx_vadd_h, src4, tmp2, src5, tmp3, src4, src5); - DUP2_ARG3(__lsx_vsrarni_b_h, src1, src0, 2, src3, src2, 2, dst0, dst1); - dst2 = __lsx_vsrarni_b_h(src5, src4, 2); - __lsx_vst(dst0, d, 0); - __lsx_vst(dst1, d, 16); - __lsx_vst(dst2, d, 32); - src_ptr += 64; - src_nex += 64; - d += 48; - } + int count = dst_width / 12; + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr0, %[lo], 0 \n" + "vld $vr1, %[hi], 0 \n" + "vld $vr2, %[wt], 0 \n" + "vld $vr3, %[pack], 0 \n" + "1: \n" + "vld $vr4, %[src], 0 \n" + "vld $vr5, %[nex], 0 \n" + "vshuf.b $vr6, $vr4, $vr4, $vr0 \n" + "vshuf.b $vr7, $vr4, $vr4, $vr1 \n" + "vshuf.b $vr8, $vr5, $vr5, $vr0 \n" + "vshuf.b $vr9, $vr5, $vr5, $vr1 \n" + "vmulwev.h.bu $vr4, $vr6, $vr2 \n" + "vmaddwod.h.bu $vr4, $vr6, $vr2 \n" + "vmulwev.h.bu $vr5, $vr7, $vr2 \n" + "vmaddwod.h.bu $vr5, $vr7, $vr2 \n" + "vmulwev.h.bu $vr6, $vr8, $vr2 \n" + "vmaddwod.h.bu $vr6, $vr8, $vr2 \n" + "vmulwev.h.bu $vr7, $vr9, $vr2 \n" + "vmaddwod.h.bu $vr7, $vr9, $vr2 \n" + "vsrari.h $vr4, $vr4, 2 \n" + "vsrari.h $vr5, $vr5, 2 \n" + "vsrari.h $vr6, $vr6, 2 \n" + "vsrari.h $vr7, $vr7, 2 \n" + "vslli.h $vr8, $vr4, 1 \n" + "vslli.h $vr9, $vr5, 1 \n" + "vadd.h $vr4, $vr4, $vr8 \n" + "vadd.h $vr5, $vr5, $vr9 \n" + "vadd.h $vr4, $vr4, $vr6 \n" + "vadd.h $vr5, $vr5, $vr7 \n" + "vsrarni.b.h $vr5, $vr4, 2 \n" + "vshuf.b $vr5, $vr5, $vr5, $vr3 \n" + "vstelm.d $vr5, %[dst], 0, 0 \n" + "vstelm.w $vr5, %[dst], 8, 2 \n" + "addi.d %[src], %[src], 16 \n" + "addi.d %[nex], %[nex], 16 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [nex] "+r"(src_nex), // %[nex] + [dst] "+r"(d), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [lo] "r"(kScale34BoxLo), // %[lo] + [hi] "r"(kScale34BoxHi), // %[hi] + [wt] "r"(kScale34BoxWt), // %[wt] + [pack] "r"(kScale34Pack12) // %[pack] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9"); } void ScaleRowDown34_1_Box_LSX(const uint8_t* src_ptr, @@ -678,57 +856,54 @@ void ScaleRowDown34_1_Box_LSX(const uint8_t* src_ptr, uint8_t* d, int dst_width) { const uint8_t* src_nex = src_ptr + src_stride; - int x; - __m128i src0, src1, src2, src3, src4, src5, src6, src7; - __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, tmp9; - __m128i tmp10, tmp11, dst0, dst1, dst2; - __m128i const0 = {0x0103030101010103, 0x0101010303010101}; - __m128i const1 = {0x0301010101030301, 0x0103030101010103}; - __m128i const2 = {0x0101010303010101, 0x0301010101030301}; - __m128i shuff0 = {0x0504030202010100, 0x0A09090807060605}; - __m128i shuff1 = {0x0F0E0E0D0D0C0B0A, 0x1514131212111110}; - __m128i shuff2 = {0x0A09090807060605, 0x0F0E0E0D0D0C0B0A}; - __m128i shift0 = {0x0002000200010002, 0x0001000200020001}; - __m128i shift1 = {0x0002000100020002, 0x0002000200010002}; - __m128i shift2 = {0x0001000200020001, 0x0002000100020002}; - - assert((dst_width % 3 == 0) && (dst_width > 0)); - - for (x = 0; x < dst_width; x += 48) { - DUP4_ARG2(__lsx_vld, src_ptr, 0, src_ptr, 16, src_ptr, 32, src_ptr, 48, - src0, src1, src2, src3); - DUP4_ARG2(__lsx_vld, src_nex, 0, src_nex, 16, src_nex, 32, src_nex, 48, - src4, src5, src6, src7); - DUP4_ARG3(__lsx_vshuf_b, src0, src0, shuff0, src1, src0, shuff1, src1, src1, - shuff2, src2, src2, shuff0, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG3(__lsx_vshuf_b, src3, src2, shuff1, src3, src3, shuff2, src4, src4, - shuff0, src5, src4, shuff1, tmp4, tmp5, tmp6, tmp7); - DUP4_ARG3(__lsx_vshuf_b, src5, src5, shuff2, src6, src6, shuff0, src7, src6, - shuff1, src7, src7, shuff2, tmp8, tmp9, tmp10, tmp11); - DUP4_ARG2(__lsx_vdp2_h_bu, tmp0, const0, tmp1, const1, tmp2, const2, tmp3, - const0, src0, src1, src2, src3); - DUP4_ARG2(__lsx_vdp2_h_bu, tmp4, const1, tmp5, const2, tmp6, const0, tmp7, - const1, src4, src5, src6, src7); - DUP4_ARG2(__lsx_vdp2_h_bu, tmp8, const2, tmp9, const0, tmp10, const1, tmp11, - const2, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG2(__lsx_vsrar_h, src0, shift0, src1, shift1, src2, shift2, src3, - shift0, src0, src1, src2, src3); - DUP4_ARG2(__lsx_vsrar_h, src4, shift1, src5, shift2, src6, shift0, src7, - shift1, src4, src5, src6, src7); - DUP4_ARG2(__lsx_vsrar_h, tmp0, shift2, tmp1, shift0, tmp2, shift1, tmp3, - shift2, tmp0, tmp1, tmp2, tmp3); - DUP4_ARG2(__lsx_vadd_h, src0, src6, src1, src7, src2, tmp0, src3, tmp1, - src0, src1, src2, src3); - DUP2_ARG2(__lsx_vadd_h, src4, tmp2, src5, tmp3, src4, src5); - DUP2_ARG3(__lsx_vsrarni_b_h, src1, src0, 1, src3, src2, 1, dst0, dst1); - dst2 = __lsx_vsrarni_b_h(src5, src4, 1); - __lsx_vst(dst0, d, 0); - __lsx_vst(dst1, d, 16); - __lsx_vst(dst2, d, 32); - src_ptr += 64; - src_nex += 64; - d += 48; - } + int count = dst_width / 12; + asm volatile( + "beqz %[cnt], 2f \n" + "vld $vr0, %[lo], 0 \n" + "vld $vr1, %[hi], 0 \n" + "vld $vr2, %[wt], 0 \n" + "vld $vr3, %[pack], 0 \n" + "1: \n" + "vld $vr4, %[src], 0 \n" + "vld $vr5, %[nex], 0 \n" + "vshuf.b $vr6, $vr4, $vr4, $vr0 \n" + "vshuf.b $vr7, $vr4, $vr4, $vr1 \n" + "vshuf.b $vr8, $vr5, $vr5, $vr0 \n" + "vshuf.b $vr9, $vr5, $vr5, $vr1 \n" + "vmulwev.h.bu $vr4, $vr6, $vr2 \n" + "vmaddwod.h.bu $vr4, $vr6, $vr2 \n" + "vmulwev.h.bu $vr5, $vr7, $vr2 \n" + "vmaddwod.h.bu $vr5, $vr7, $vr2 \n" + "vmulwev.h.bu $vr6, $vr8, $vr2 \n" + "vmaddwod.h.bu $vr6, $vr8, $vr2 \n" + "vmulwev.h.bu $vr7, $vr9, $vr2 \n" + "vmaddwod.h.bu $vr7, $vr9, $vr2 \n" + "vsrari.h $vr4, $vr4, 2 \n" + "vsrari.h $vr5, $vr5, 2 \n" + "vsrari.h $vr6, $vr6, 2 \n" + "vsrari.h $vr7, $vr7, 2 \n" + "vadd.h $vr4, $vr4, $vr6 \n" + "vadd.h $vr5, $vr5, $vr7 \n" + "vsrarni.b.h $vr5, $vr4, 1 \n" + "vshuf.b $vr5, $vr5, $vr5, $vr3 \n" + "vstelm.d $vr5, %[dst], 0, 0 \n" + "vstelm.w $vr5, %[dst], 8, 2 \n" + "addi.d %[src], %[src], 16 \n" + "addi.d %[nex], %[nex], 16 \n" + "addi.d %[dst], %[dst], 12 \n" + "addi.w %[cnt], %[cnt], -1 \n" + "bnez %[cnt], 1b \n" + "2: \n" + : [src] "+r"(src_ptr), // %[src] + [nex] "+r"(src_nex), // %[nex] + [dst] "+r"(d), // %[dst] + [cnt] "+r"(count) // %[cnt] + : [lo] "r"(kScale34BoxLo), // %[lo] + [hi] "r"(kScale34BoxHi), // %[hi] + [wt] "r"(kScale34BoxWt), // %[wt] + [pack] "r"(kScale34Pack12) // %[pack] + : "memory", "$vr0", "$vr1", "$vr2", "$vr3", "$vr4", "$vr5", "$vr6", + "$vr7", "$vr8", "$vr9"); } #ifdef __cplusplus diff --git a/source/scale_uv.cc b/source/scale_uv.cc index 90dc8b2ce..4c0b1065a 100644 --- a/source/scale_uv.cc +++ b/source/scale_uv.cc @@ -547,11 +547,19 @@ static int ScaleUVBilinearDown(int src_width, #if defined(HAS_INTERPOLATEROW_LSX) if (TestCpuFlag(kCpuHasLSX)) { InterpolateRow = InterpolateRow_Any_LSX; - if (IS_ALIGNED(clip_src_width, 32)) { + if (IS_ALIGNED(clip_src_width, 16)) { InterpolateRow = InterpolateRow_LSX; } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(clip_src_width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV; @@ -678,6 +686,14 @@ static int ScaleUVBilinearUp(int src_width, } } #endif +#if defined(HAS_INTERPOLATEROW_LASX) + if (TestCpuFlag(kCpuHasLASX)) { + InterpolateRow = InterpolateRow_Any_LASX; + if (IS_ALIGNED(dst_width, 32)) { + InterpolateRow = InterpolateRow_LASX; + } + } +#endif #if defined(HAS_INTERPOLATEROW_RVV) if (TestCpuFlag(kCpuHasRVV)) { InterpolateRow = InterpolateRow_RVV;