Go to the documentation of this file. 21 #ifndef AVUTIL_MIPS_GENERIC_MACROS_MSA_H 22 #define AVUTIL_MIPS_GENERIC_MACROS_MSA_H 28 #define ALLOC_ALIGNED(align) __attribute__ ((aligned((align) << 1))) 30 #define LD_V(RTYPE, psrc) *((RTYPE *)(psrc)) 31 #define LD_UB(...) LD_V(v16u8, __VA_ARGS__) 32 #define LD_SB(...) LD_V(v16i8, __VA_ARGS__) 33 #define LD_UH(...) LD_V(v8u16, __VA_ARGS__) 34 #define LD_SH(...) LD_V(v8i16, __VA_ARGS__) 35 #define LD_UW(...) LD_V(v4u32, __VA_ARGS__) 36 #define LD_SW(...) LD_V(v4i32, __VA_ARGS__) 38 #define ST_V(RTYPE, in, pdst) *((RTYPE *)(pdst)) = (in) 39 #define ST_UB(...) ST_V(v16u8, __VA_ARGS__) 40 #define ST_SB(...) ST_V(v16i8, __VA_ARGS__) 41 #define ST_UH(...) ST_V(v8u16, __VA_ARGS__) 42 #define ST_SH(...) ST_V(v8i16, __VA_ARGS__) 43 #define ST_UW(...) ST_V(v4u32, __VA_ARGS__) 44 #define ST_SW(...) ST_V(v4i32, __VA_ARGS__) 46 #if (__mips_isa_rev >= 6) 49 uint16_t val_lh_m = *(uint16_t *)(psrc); \ 55 uint32_t val_lw_m = *(uint32_t *)(psrc); \ 62 uint64_t val_ld_m = *(uint64_t *)(psrc); \ 65 #else // !(__mips == 64) 68 uint8_t *psrc_ld_m = (uint8_t *) (psrc); \ 69 uint32_t val0_ld_m, val1_ld_m; \ 70 uint64_t val_ld_m = 0; \ 72 val0_ld_m = LW(psrc_ld_m); \ 73 val1_ld_m = LW(psrc_ld_m + 4); \ 75 val_ld_m = (uint64_t) (val1_ld_m); \ 76 val_ld_m = (uint64_t) ((val_ld_m << 32) & 0xFFFFFFFF00000000); \ 77 val_ld_m = (uint64_t) (val_ld_m | (uint64_t) val0_ld_m); \ 81 #endif // (__mips == 64) 83 #define SH(val, pdst) *(uint16_t *)(pdst) = (val); 84 #define SW(val, pdst) *(uint32_t *)(pdst) = (val); 85 #define SD(val, pdst) *(uint64_t *)(pdst) = (val); 87 #else // !(__mips_isa_rev >= 6) 90 uint8_t *psrc_lh_m = (uint8_t *) (psrc); \ 94 "ulh %[val_lh_m], %[psrc_lh_m] \n\t" \ 96 : [val_lh_m] "=r" (val_lh_m) \ 97 : [psrc_lh_m] "m" (*psrc_lh_m) \ 105 uint8_t *psrc_lw_m = (uint8_t *) (psrc); \ 109 "ulw %[val_lw_m], %[psrc_lw_m] \n\t" \ 111 : [val_lw_m] "=r" (val_lw_m) \ 112 : [psrc_lw_m] "m" (*psrc_lw_m) \ 121 uint8_t *psrc_ld_m = (uint8_t *) (psrc); \ 122 uint64_t val_ld_m = 0; \ 125 "uld %[val_ld_m], %[psrc_ld_m] \n\t" \ 127 : [val_ld_m] "=r" (val_ld_m) \ 128 : [psrc_ld_m] "m" (*psrc_ld_m) \ 133 #else // !(__mips == 64) 136 uint8_t *psrc_ld_m = (uint8_t *) (psrc); \ 137 uint32_t val0_ld_m, val1_ld_m; \ 138 uint64_t val_ld_m = 0; \ 140 val0_ld_m = LW(psrc_ld_m); \ 141 val1_ld_m = LW(psrc_ld_m + 4); \ 143 val_ld_m = (uint64_t) (val1_ld_m); \ 144 val_ld_m = (uint64_t) ((val_ld_m << 32) & 0xFFFFFFFF00000000); \ 145 val_ld_m = (uint64_t) (val_ld_m | (uint64_t) val0_ld_m); \ 149 #endif // (__mips == 64) 151 #define SH(val, pdst) \ 153 uint8_t *pdst_sh_m = (uint8_t *) (pdst); \ 154 uint16_t val_sh_m = (val); \ 157 "ush %[val_sh_m], %[pdst_sh_m] \n\t" \ 159 : [pdst_sh_m] "=m" (*pdst_sh_m) \ 160 : [val_sh_m] "r" (val_sh_m) \ 164 #define SW(val, pdst) \ 166 uint8_t *pdst_sw_m = (uint8_t *) (pdst); \ 167 uint32_t val_sw_m = (val); \ 170 "usw %[val_sw_m], %[pdst_sw_m] \n\t" \ 172 : [pdst_sw_m] "=m" (*pdst_sw_m) \ 173 : [val_sw_m] "r" (val_sw_m) \ 177 #define SD(val, pdst) \ 179 uint8_t *pdst_sd_m = (uint8_t *) (pdst); \ 180 uint32_t val0_sd_m, val1_sd_m; \ 182 val0_sd_m = (uint32_t) ((val) & 0x00000000FFFFFFFF); \ 183 val1_sd_m = (uint32_t) (((val) >> 32) & 0x00000000FFFFFFFF); \ 185 SW(val0_sd_m, pdst_sd_m); \ 186 SW(val1_sd_m, pdst_sd_m + 4); \ 188 #endif // (__mips_isa_rev >= 6) 199 #define LW4(psrc, stride, out0, out1, out2, out3) \ 202 out1 = LW((psrc) + stride); \ 203 out2 = LW((psrc) + 2 * stride); \ 204 out3 = LW((psrc) + 3 * stride); \ 207 #define LW2(psrc, stride, out0, out1) \ 210 out1 = LW((psrc) + stride); \ 220 #define LD2(psrc, stride, out0, out1) \ 223 out1 = LD((psrc) + stride); \ 225 #define LD4(psrc, stride, out0, out1, out2, out3) \ 227 LD2((psrc), stride, out0, out1); \ 228 LD2((psrc) + 2 * stride, stride, out2, out3); \ 238 #define SW4(in0, in1, in2, in3, pdst, stride) \ 241 SW(in1, (pdst) + stride); \ 242 SW(in2, (pdst) + 2 * stride); \ 243 SW(in3, (pdst) + 3 * stride); \ 253 #define SD4(in0, in1, in2, in3, pdst, stride) \ 256 SD(in1, (pdst) + stride); \ 257 SD(in2, (pdst) + 2 * stride); \ 258 SD(in3, (pdst) + 3 * stride); \ 269 #define LD_V2(RTYPE, psrc, stride, out0, out1) \ 271 out0 = LD_V(RTYPE, (psrc)); \ 272 out1 = LD_V(RTYPE, (psrc) + stride); \ 274 #define LD_UB2(...) LD_V2(v16u8, __VA_ARGS__) 275 #define LD_SB2(...) LD_V2(v16i8, __VA_ARGS__) 276 #define LD_UH2(...) LD_V2(v8u16, __VA_ARGS__) 277 #define LD_SH2(...) LD_V2(v8i16, __VA_ARGS__) 278 #define LD_SW2(...) LD_V2(v4i32, __VA_ARGS__) 280 #define LD_V3(RTYPE, psrc, stride, out0, out1, out2) \ 282 LD_V2(RTYPE, (psrc), stride, out0, out1); \ 283 out2 = LD_V(RTYPE, (psrc) + 2 * stride); \ 285 #define LD_UB3(...) LD_V3(v16u8, __VA_ARGS__) 286 #define LD_SB3(...) LD_V3(v16i8, __VA_ARGS__) 288 #define LD_V4(RTYPE, psrc, stride, out0, out1, out2, out3) \ 290 LD_V2(RTYPE, (psrc), stride, out0, out1); \ 291 LD_V2(RTYPE, (psrc) + 2 * stride , stride, out2, out3); \ 293 #define LD_UB4(...) LD_V4(v16u8, __VA_ARGS__) 294 #define LD_SB4(...) LD_V4(v16i8, __VA_ARGS__) 295 #define LD_UH4(...) LD_V4(v8u16, __VA_ARGS__) 296 #define LD_SH4(...) LD_V4(v8i16, __VA_ARGS__) 298 #define LD_V5(RTYPE, psrc, stride, out0, out1, out2, out3, out4) \ 300 LD_V4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ 301 out4 = LD_V(RTYPE, (psrc) + 4 * stride); \ 303 #define LD_UB5(...) LD_V5(v16u8, __VA_ARGS__) 304 #define LD_SB5(...) LD_V5(v16i8, __VA_ARGS__) 306 #define LD_V6(RTYPE, psrc, stride, out0, out1, out2, out3, out4, out5) \ 308 LD_V4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ 309 LD_V2(RTYPE, (psrc) + 4 * stride, stride, out4, out5); \ 311 #define LD_UB6(...) LD_V6(v16u8, __VA_ARGS__) 312 #define LD_SB6(...) LD_V6(v16i8, __VA_ARGS__) 313 #define LD_UH6(...) LD_V6(v8u16, __VA_ARGS__) 314 #define LD_SH6(...) LD_V6(v8i16, __VA_ARGS__) 316 #define LD_V7(RTYPE, psrc, stride, \ 317 out0, out1, out2, out3, out4, out5, out6) \ 319 LD_V5(RTYPE, (psrc), stride, out0, out1, out2, out3, out4); \ 320 LD_V2(RTYPE, (psrc) + 5 * stride, stride, out5, out6); \ 322 #define LD_UB7(...) LD_V7(v16u8, __VA_ARGS__) 323 #define LD_SB7(...) LD_V7(v16i8, __VA_ARGS__) 325 #define LD_V8(RTYPE, psrc, stride, \ 326 out0, out1, out2, out3, out4, out5, out6, out7) \ 328 LD_V4(RTYPE, (psrc), stride, out0, out1, out2, out3); \ 329 LD_V4(RTYPE, (psrc) + 4 * stride, stride, out4, out5, out6, out7); \ 331 #define LD_UB8(...) LD_V8(v16u8, __VA_ARGS__) 332 #define LD_SB8(...) LD_V8(v16i8, __VA_ARGS__) 333 #define LD_UH8(...) LD_V8(v8u16, __VA_ARGS__) 334 #define LD_SH8(...) LD_V8(v8i16, __VA_ARGS__) 336 #define LD_V16(RTYPE, psrc, stride, \ 337 out0, out1, out2, out3, out4, out5, out6, out7, \ 338 out8, out9, out10, out11, out12, out13, out14, out15) \ 340 LD_V8(RTYPE, (psrc), stride, \ 341 out0, out1, out2, out3, out4, out5, out6, out7); \ 342 LD_V8(RTYPE, (psrc) + 8 * stride, stride, \ 343 out8, out9, out10, out11, out12, out13, out14, out15); \ 345 #define LD_SH16(...) LD_V16(v8i16, __VA_ARGS__) 352 #define LD4x4_SH(psrc, out0, out1, out2, out3) \ 354 out0 = LD_SH(psrc); \ 355 out2 = LD_SH(psrc + 8); \ 356 out1 = (v8i16) __msa_ilvl_d((v2i64) out0, (v2i64) out0); \ 357 out3 = (v8i16) __msa_ilvl_d((v2i64) out2, (v2i64) out2); \ 366 #define ST_V2(RTYPE, in0, in1, pdst, stride) \ 368 ST_V(RTYPE, in0, (pdst)); \ 369 ST_V(RTYPE, in1, (pdst) + stride); \ 371 #define ST_UB2(...) ST_V2(v16u8, __VA_ARGS__) 372 #define ST_SB2(...) ST_V2(v16i8, __VA_ARGS__) 373 #define ST_UH2(...) ST_V2(v8u16, __VA_ARGS__) 374 #define ST_SH2(...) ST_V2(v8i16, __VA_ARGS__) 375 #define ST_SW2(...) ST_V2(v4i32, __VA_ARGS__) 377 #define ST_V4(RTYPE, in0, in1, in2, in3, pdst, stride) \ 379 ST_V2(RTYPE, in0, in1, (pdst), stride); \ 380 ST_V2(RTYPE, in2, in3, (pdst) + 2 * stride, stride); \ 382 #define ST_UB4(...) ST_V4(v16u8, __VA_ARGS__) 383 #define ST_SB4(...) ST_V4(v16i8, __VA_ARGS__) 384 #define ST_SH4(...) ST_V4(v8i16, __VA_ARGS__) 385 #define ST_SW4(...) ST_V4(v4i32, __VA_ARGS__) 387 #define ST_V6(RTYPE, in0, in1, in2, in3, in4, in5, pdst, stride) \ 389 ST_V4(RTYPE, in0, in1, in2, in3, (pdst), stride); \ 390 ST_V2(RTYPE, in4, in5, (pdst) + 4 * stride, stride); \ 392 #define ST_SH6(...) ST_V6(v8i16, __VA_ARGS__) 394 #define ST_V8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 396 ST_V4(RTYPE, in0, in1, in2, in3, (pdst), stride); \ 397 ST_V4(RTYPE, in4, in5, in6, in7, (pdst) + 4 * stride, stride); \ 399 #define ST_UB8(...) ST_V8(v16u8, __VA_ARGS__) 400 #define ST_SH8(...) ST_V8(v8i16, __VA_ARGS__) 401 #define ST_SW8(...) ST_V8(v4i32, __VA_ARGS__) 415 #define ST2x4_UB(in, stidx, pdst, stride) \ 417 uint16_t out0_m, out1_m, out2_m, out3_m; \ 418 uint8_t *pblk_2x4_m = (uint8_t *) (pdst); \ 420 out0_m = __msa_copy_u_h((v8i16) in, (stidx)); \ 421 out1_m = __msa_copy_u_h((v8i16) in, (stidx + 1)); \ 422 out2_m = __msa_copy_u_h((v8i16) in, (stidx + 2)); \ 423 out3_m = __msa_copy_u_h((v8i16) in, (stidx + 3)); \ 425 SH(out0_m, pblk_2x4_m); \ 426 SH(out1_m, pblk_2x4_m + stride); \ 427 SH(out2_m, pblk_2x4_m + 2 * stride); \ 428 SH(out3_m, pblk_2x4_m + 3 * stride); \ 439 #define ST4x2_UB(in, pdst, stride) \ 441 uint32_t out0_m, out1_m; \ 442 uint8_t *pblk_4x2_m = (uint8_t *) (pdst); \ 444 out0_m = __msa_copy_u_w((v4i32) in, 0); \ 445 out1_m = __msa_copy_u_w((v4i32) in, 1); \ 447 SW(out0_m, pblk_4x2_m); \ 448 SW(out1_m, pblk_4x2_m + stride); \ 463 #define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride) \ 465 uint32_t out0_m, out1_m, out2_m, out3_m; \ 466 uint8_t *pblk_4x4_m = (uint8_t *) (pdst); \ 468 out0_m = __msa_copy_u_w((v4i32) in0, idx0); \ 469 out1_m = __msa_copy_u_w((v4i32) in0, idx1); \ 470 out2_m = __msa_copy_u_w((v4i32) in1, idx2); \ 471 out3_m = __msa_copy_u_w((v4i32) in1, idx3); \ 473 SW4(out0_m, out1_m, out2_m, out3_m, pblk_4x4_m, stride); \ 475 #define ST4x8_UB(in0, in1, pdst, stride) \ 477 uint8_t *pblk_4x8 = (uint8_t *) (pdst); \ 479 ST4x4_UB(in0, in0, 0, 1, 2, 3, pblk_4x8, stride); \ 480 ST4x4_UB(in1, in1, 0, 1, 2, 3, pblk_4x8 + 4 * stride, stride); \ 496 #define ST6x4_UB(in0, in1, pdst, stride) \ 498 uint32_t out0_m, out1_m, out2_m, out3_m; \ 499 uint16_t out4_m, out5_m, out6_m, out7_m; \ 500 uint8_t *pblk_6x4_m = (uint8_t *) (pdst); \ 502 out0_m = __msa_copy_u_w((v4i32) in0, 0); \ 503 out1_m = __msa_copy_u_w((v4i32) in0, 2); \ 504 out2_m = __msa_copy_u_w((v4i32) in1, 0); \ 505 out3_m = __msa_copy_u_w((v4i32) in1, 2); \ 507 out4_m = __msa_copy_u_h((v8i16) in0, 2); \ 508 out5_m = __msa_copy_u_h((v8i16) in0, 6); \ 509 out6_m = __msa_copy_u_h((v8i16) in1, 2); \ 510 out7_m = __msa_copy_u_h((v8i16) in1, 6); \ 512 SW(out0_m, pblk_6x4_m); \ 513 SH(out4_m, (pblk_6x4_m + 4)); \ 514 pblk_6x4_m += stride; \ 515 SW(out1_m, pblk_6x4_m); \ 516 SH(out5_m, (pblk_6x4_m + 4)); \ 517 pblk_6x4_m += stride; \ 518 SW(out2_m, pblk_6x4_m); \ 519 SH(out6_m, (pblk_6x4_m + 4)); \ 520 pblk_6x4_m += stride; \ 521 SW(out3_m, pblk_6x4_m); \ 522 SH(out7_m, (pblk_6x4_m + 4)); \ 530 #define ST8x1_UB(in, pdst) \ 533 out0_m = __msa_copy_u_d((v2i64) in, 0); \ 544 #define ST8x2_UB(in, pdst, stride) \ 546 uint64_t out0_m, out1_m; \ 547 uint8_t *pblk_8x2_m = (uint8_t *) (pdst); \ 549 out0_m = __msa_copy_u_d((v2i64) in, 0); \ 550 out1_m = __msa_copy_u_d((v2i64) in, 1); \ 552 SD(out0_m, pblk_8x2_m); \ 553 SD(out1_m, pblk_8x2_m + stride); \ 568 #define ST8x4_UB(in0, in1, pdst, stride) \ 570 uint64_t out0_m, out1_m, out2_m, out3_m; \ 571 uint8_t *pblk_8x4_m = (uint8_t *) (pdst); \ 573 out0_m = __msa_copy_u_d((v2i64) in0, 0); \ 574 out1_m = __msa_copy_u_d((v2i64) in0, 1); \ 575 out2_m = __msa_copy_u_d((v2i64) in1, 0); \ 576 out3_m = __msa_copy_u_d((v2i64) in1, 1); \ 578 SD4(out0_m, out1_m, out2_m, out3_m, pblk_8x4_m, stride); \ 580 #define ST8x8_UB(in0, in1, in2, in3, pdst, stride) \ 582 uint8_t *pblk_8x8_m = (uint8_t *) (pdst); \ 584 ST8x4_UB(in0, in1, pblk_8x8_m, stride); \ 585 ST8x4_UB(in2, in3, pblk_8x8_m + 4 * stride, stride); \ 587 #define ST12x4_UB(in0, in1, in2, pdst, stride) \ 589 uint8_t *pblk_12x4_m = (uint8_t *) (pdst); \ 592 ST8x4_UB(in0, in1, pblk_12x4_m, stride); \ 594 ST4x4_UB(in2, in2, 0, 1, 2, 3, pblk_12x4_m + 8, stride); \ 606 #define ST12x8_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 608 uint64_t out0_m, out1_m, out2_m, out3_m; \ 609 uint64_t out4_m, out5_m, out6_m, out7_m; \ 610 uint32_t out8_m, out9_m, out10_m, out11_m; \ 611 uint32_t out12_m, out13_m, out14_m, out15_m; \ 612 uint8_t *pblk_12x8_m = (uint8_t *) (pdst); \ 614 out0_m = __msa_copy_u_d((v2i64) in0, 0); \ 615 out1_m = __msa_copy_u_d((v2i64) in1, 0); \ 616 out2_m = __msa_copy_u_d((v2i64) in2, 0); \ 617 out3_m = __msa_copy_u_d((v2i64) in3, 0); \ 618 out4_m = __msa_copy_u_d((v2i64) in4, 0); \ 619 out5_m = __msa_copy_u_d((v2i64) in5, 0); \ 620 out6_m = __msa_copy_u_d((v2i64) in6, 0); \ 621 out7_m = __msa_copy_u_d((v2i64) in7, 0); \ 623 out8_m = __msa_copy_u_w((v4i32) in0, 2); \ 624 out9_m = __msa_copy_u_w((v4i32) in1, 2); \ 625 out10_m = __msa_copy_u_w((v4i32) in2, 2); \ 626 out11_m = __msa_copy_u_w((v4i32) in3, 2); \ 627 out12_m = __msa_copy_u_w((v4i32) in4, 2); \ 628 out13_m = __msa_copy_u_w((v4i32) in5, 2); \ 629 out14_m = __msa_copy_u_w((v4i32) in6, 2); \ 630 out15_m = __msa_copy_u_w((v4i32) in7, 2); \ 632 SD(out0_m, pblk_12x8_m); \ 633 SW(out8_m, pblk_12x8_m + 8); \ 634 pblk_12x8_m += stride; \ 635 SD(out1_m, pblk_12x8_m); \ 636 SW(out9_m, pblk_12x8_m + 8); \ 637 pblk_12x8_m += stride; \ 638 SD(out2_m, pblk_12x8_m); \ 639 SW(out10_m, pblk_12x8_m + 8); \ 640 pblk_12x8_m += stride; \ 641 SD(out3_m, pblk_12x8_m); \ 642 SW(out11_m, pblk_12x8_m + 8); \ 643 pblk_12x8_m += stride; \ 644 SD(out4_m, pblk_12x8_m); \ 645 SW(out12_m, pblk_12x8_m + 8); \ 646 pblk_12x8_m += stride; \ 647 SD(out5_m, pblk_12x8_m); \ 648 SW(out13_m, pblk_12x8_m + 8); \ 649 pblk_12x8_m += stride; \ 650 SD(out6_m, pblk_12x8_m); \ 651 SW(out14_m, pblk_12x8_m + 8); \ 652 pblk_12x8_m += stride; \ 653 SD(out7_m, pblk_12x8_m); \ 654 SW(out15_m, pblk_12x8_m + 8); \ 669 #define AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) \ 671 out0 = (RTYPE) __msa_aver_u_b((v16u8) in0, (v16u8) in1); \ 672 out1 = (RTYPE) __msa_aver_u_b((v16u8) in2, (v16u8) in3); \ 674 #define AVER_UB2_UB(...) AVER_UB2(v16u8, __VA_ARGS__) 676 #define AVER_UB4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 677 out0, out1, out2, out3) \ 679 AVER_UB2(RTYPE, in0, in1, in2, in3, out0, out1) \ 680 AVER_UB2(RTYPE, in4, in5, in6, in7, out2, out3) \ 682 #define AVER_UB4_UB(...) AVER_UB4(v16u8, __VA_ARGS__) 691 #define SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val) \ 693 v16i8 zero_m = { 0 }; \ 694 out0 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in0, slide_val); \ 695 out1 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in1, slide_val); \ 697 #define SLDI_B2_0_UB(...) SLDI_B2_0(v16u8, __VA_ARGS__) 698 #define SLDI_B2_0_SB(...) SLDI_B2_0(v16i8, __VA_ARGS__) 699 #define SLDI_B2_0_SW(...) SLDI_B2_0(v4i32, __VA_ARGS__) 701 #define SLDI_B3_0(RTYPE, in0, in1, in2, out0, out1, out2, slide_val) \ 703 v16i8 zero_m = { 0 }; \ 704 SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val); \ 705 out2 = (RTYPE) __msa_sldi_b((v16i8) zero_m, (v16i8) in2, slide_val); \ 707 #define SLDI_B3_0_UB(...) SLDI_B3_0(v16u8, __VA_ARGS__) 708 #define SLDI_B3_0_SB(...) SLDI_B3_0(v16i8, __VA_ARGS__) 710 #define SLDI_B4_0(RTYPE, in0, in1, in2, in3, \ 711 out0, out1, out2, out3, slide_val) \ 713 SLDI_B2_0(RTYPE, in0, in1, out0, out1, slide_val); \ 714 SLDI_B2_0(RTYPE, in2, in3, out2, out3, slide_val); \ 716 #define SLDI_B4_0_UB(...) SLDI_B4_0(v16u8, __VA_ARGS__) 717 #define SLDI_B4_0_SB(...) SLDI_B4_0(v16i8, __VA_ARGS__) 718 #define SLDI_B4_0_SH(...) SLDI_B4_0(v8i16, __VA_ARGS__) 727 #define SLDI_B2(RTYPE, in0_0, in0_1, in1_0, in1_1, out0, out1, slide_val) \ 729 out0 = (RTYPE) __msa_sldi_b((v16i8) in0_0, (v16i8) in1_0, slide_val); \ 730 out1 = (RTYPE) __msa_sldi_b((v16i8) in0_1, (v16i8) in1_1, slide_val); \ 732 #define SLDI_B2_UB(...) SLDI_B2(v16u8, __VA_ARGS__) 733 #define SLDI_B2_SB(...) SLDI_B2(v16i8, __VA_ARGS__) 734 #define SLDI_B2_SH(...) SLDI_B2(v8i16, __VA_ARGS__) 736 #define SLDI_B3(RTYPE, in0_0, in0_1, in0_2, in1_0, in1_1, in1_2, \ 737 out0, out1, out2, slide_val) \ 739 SLDI_B2(RTYPE, in0_0, in0_1, in1_0, in1_1, out0, out1, slide_val) \ 740 out2 = (RTYPE) __msa_sldi_b((v16i8) in0_2, (v16i8) in1_2, slide_val); \ 742 #define SLDI_B3_SB(...) SLDI_B3(v16i8, __VA_ARGS__) 743 #define SLDI_B3_UH(...) SLDI_B3(v8u16, __VA_ARGS__) 754 #define VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \ 756 out0 = (RTYPE) __msa_vshf_b((v16i8) mask0, (v16i8) in1, (v16i8) in0); \ 757 out1 = (RTYPE) __msa_vshf_b((v16i8) mask1, (v16i8) in3, (v16i8) in2); \ 759 #define VSHF_B2_UB(...) VSHF_B2(v16u8, __VA_ARGS__) 760 #define VSHF_B2_SB(...) VSHF_B2(v16i8, __VA_ARGS__) 761 #define VSHF_B2_UH(...) VSHF_B2(v8u16, __VA_ARGS__) 762 #define VSHF_B2_SH(...) VSHF_B2(v8i16, __VA_ARGS__) 764 #define VSHF_B3(RTYPE, in0, in1, in2, in3, in4, in5, mask0, mask1, mask2, \ 767 VSHF_B2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1); \ 768 out2 = (RTYPE) __msa_vshf_b((v16i8) mask2, (v16i8) in5, (v16i8) in4); \ 770 #define VSHF_B3_SB(...) VSHF_B3(v16i8, __VA_ARGS__) 772 #define VSHF_B4(RTYPE, in0, in1, mask0, mask1, mask2, mask3, \ 773 out0, out1, out2, out3) \ 775 VSHF_B2(RTYPE, in0, in1, in0, in1, mask0, mask1, out0, out1); \ 776 VSHF_B2(RTYPE, in0, in1, in0, in1, mask2, mask3, out2, out3); \ 778 #define VSHF_B4_SB(...) VSHF_B4(v16i8, __VA_ARGS__) 779 #define VSHF_B4_SH(...) VSHF_B4(v8i16, __VA_ARGS__) 790 #define VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \ 792 out0 = (RTYPE) __msa_vshf_h((v8i16) mask0, (v8i16) in1, (v8i16) in0); \ 793 out1 = (RTYPE) __msa_vshf_h((v8i16) mask1, (v8i16) in3, (v8i16) in2); \ 795 #define VSHF_H2_SH(...) VSHF_H2(v8i16, __VA_ARGS__) 797 #define VSHF_H3(RTYPE, in0, in1, in2, in3, in4, in5, mask0, mask1, mask2, \ 800 VSHF_H2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1); \ 801 out2 = (RTYPE) __msa_vshf_h((v8i16) mask2, (v8i16) in5, (v8i16) in4); \ 803 #define VSHF_H3_SH(...) VSHF_H3(v8i16, __VA_ARGS__) 814 #define VSHF_W2(RTYPE, in0, in1, in2, in3, mask0, mask1, out0, out1) \ 816 out0 = (RTYPE) __msa_vshf_w((v4i32) mask0, (v4i32) in1, (v4i32) in0); \ 817 out1 = (RTYPE) __msa_vshf_w((v4i32) mask1, (v4i32) in3, (v4i32) in2); \ 819 #define VSHF_W2_SB(...) VSHF_W2(v16i8, __VA_ARGS__) 833 #define DOTP_UB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 835 out0 = (RTYPE) __msa_dotp_u_h((v16u8) mult0, (v16u8) cnst0); \ 836 out1 = (RTYPE) __msa_dotp_u_h((v16u8) mult1, (v16u8) cnst1); \ 838 #define DOTP_UB2_UH(...) DOTP_UB2(v8u16, __VA_ARGS__) 840 #define DOTP_UB4(RTYPE, mult0, mult1, mult2, mult3, \ 841 cnst0, cnst1, cnst2, cnst3, \ 842 out0, out1, out2, out3) \ 844 DOTP_UB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 845 DOTP_UB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 847 #define DOTP_UB4_UH(...) DOTP_UB4(v8u16, __VA_ARGS__) 861 #define DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 863 out0 = (RTYPE) __msa_dotp_s_h((v16i8) mult0, (v16i8) cnst0); \ 864 out1 = (RTYPE) __msa_dotp_s_h((v16i8) mult1, (v16i8) cnst1); \ 866 #define DOTP_SB2_SH(...) DOTP_SB2(v8i16, __VA_ARGS__) 868 #define DOTP_SB3(RTYPE, mult0, mult1, mult2, cnst0, cnst1, cnst2, \ 871 DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 872 out2 = (RTYPE) __msa_dotp_s_h((v16i8) mult2, (v16i8) cnst2); \ 874 #define DOTP_SB3_SH(...) DOTP_SB3(v8i16, __VA_ARGS__) 876 #define DOTP_SB4(RTYPE, mult0, mult1, mult2, mult3, \ 877 cnst0, cnst1, cnst2, cnst3, out0, out1, out2, out3) \ 879 DOTP_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 880 DOTP_SB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 882 #define DOTP_SB4_SH(...) DOTP_SB4(v8i16, __VA_ARGS__) 896 #define DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 898 out0 = (RTYPE) __msa_dotp_s_w((v8i16) mult0, (v8i16) cnst0); \ 899 out1 = (RTYPE) __msa_dotp_s_w((v8i16) mult1, (v8i16) cnst1); \ 901 #define DOTP_SH2_SW(...) DOTP_SH2(v4i32, __VA_ARGS__) 903 #define DOTP_SH4(RTYPE, mult0, mult1, mult2, mult3, \ 904 cnst0, cnst1, cnst2, cnst3, \ 905 out0, out1, out2, out3) \ 907 DOTP_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 908 DOTP_SH2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 910 #define DOTP_SH4_SW(...) DOTP_SH4(v4i32, __VA_ARGS__) 924 #define DPADD_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 926 out0 = (RTYPE) __msa_dpadd_s_h((v8i16) out0, \ 927 (v16i8) mult0, (v16i8) cnst0); \ 928 out1 = (RTYPE) __msa_dpadd_s_h((v8i16) out1, \ 929 (v16i8) mult1, (v16i8) cnst1); \ 931 #define DPADD_SB2_SH(...) DPADD_SB2(v8i16, __VA_ARGS__) 933 #define DPADD_SB4(RTYPE, mult0, mult1, mult2, mult3, \ 934 cnst0, cnst1, cnst2, cnst3, out0, out1, out2, out3) \ 936 DPADD_SB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 937 DPADD_SB2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 939 #define DPADD_SB4_SH(...) DPADD_SB4(v8i16, __VA_ARGS__) 953 #define DPADD_UB2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 955 out0 = (RTYPE) __msa_dpadd_u_h((v8u16) out0, \ 956 (v16u8) mult0, (v16u8) cnst0); \ 957 out1 = (RTYPE) __msa_dpadd_u_h((v8u16) out1, \ 958 (v16u8) mult1, (v16u8) cnst1); \ 960 #define DPADD_UB2_UH(...) DPADD_UB2(v8u16, __VA_ARGS__) 974 #define DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1) \ 976 out0 = (RTYPE) __msa_dpadd_s_w((v4i32) out0, \ 977 (v8i16) mult0, (v8i16) cnst0); \ 978 out1 = (RTYPE) __msa_dpadd_s_w((v4i32) out1, \ 979 (v8i16) mult1, (v8i16) cnst1); \ 981 #define DPADD_SH2_SW(...) DPADD_SH2(v4i32, __VA_ARGS__) 983 #define DPADD_SH4(RTYPE, mult0, mult1, mult2, mult3, \ 984 cnst0, cnst1, cnst2, cnst3, out0, out1, out2, out3) \ 986 DPADD_SH2(RTYPE, mult0, mult1, cnst0, cnst1, out0, out1); \ 987 DPADD_SH2(RTYPE, mult2, mult3, cnst2, cnst3, out2, out3); \ 989 #define DPADD_SH4_SW(...) DPADD_SH4(v4i32, __VA_ARGS__) 999 #define MIN_UH2(RTYPE, in0, in1, min_vec) \ 1001 in0 = (RTYPE) __msa_min_u_h((v8u16) in0, min_vec); \ 1002 in1 = (RTYPE) __msa_min_u_h((v8u16) in1, min_vec); \ 1004 #define MIN_UH2_UH(...) MIN_UH2(v8u16, __VA_ARGS__) 1006 #define MIN_UH4(RTYPE, in0, in1, in2, in3, min_vec) \ 1008 MIN_UH2(RTYPE, in0, in1, min_vec); \ 1009 MIN_UH2(RTYPE, in2, in3, min_vec); \ 1011 #define MIN_UH4_UH(...) MIN_UH4(v8u16, __VA_ARGS__) 1021 #define CLIP_SH(in, min, max) \ 1025 out_m = __msa_max_s_h((v8i16) min, (v8i16) in); \ 1026 out_m = __msa_min_s_h((v8i16) max, (v8i16) out_m); \ 1036 #define CLIP_SH_0_255(in) \ 1038 v8i16 max_m = __msa_ldi_h(255); \ 1041 out_m = __msa_maxi_s_h((v8i16) in, 0); \ 1042 out_m = __msa_min_s_h((v8i16) max_m, (v8i16) out_m); \ 1045 #define CLIP_SH2_0_255(in0, in1) \ 1047 in0 = CLIP_SH_0_255(in0); \ 1048 in1 = CLIP_SH_0_255(in1); \ 1050 #define CLIP_SH4_0_255(in0, in1, in2, in3) \ 1052 CLIP_SH2_0_255(in0, in1); \ 1053 CLIP_SH2_0_255(in2, in3); \ 1056 #define CLIP_SH_0_255_MAX_SATU(in) \ 1060 out_m = __msa_maxi_s_h((v8i16) in, 0); \ 1061 out_m = (v8i16) __msa_sat_u_h((v8u16) out_m, 7); \ 1064 #define CLIP_SH2_0_255_MAX_SATU(in0, in1) \ 1066 in0 = CLIP_SH_0_255_MAX_SATU(in0); \ 1067 in1 = CLIP_SH_0_255_MAX_SATU(in1); \ 1069 #define CLIP_SH4_0_255_MAX_SATU(in0, in1, in2, in3) \ 1071 CLIP_SH2_0_255_MAX_SATU(in0, in1); \ 1072 CLIP_SH2_0_255_MAX_SATU(in2, in3); \ 1081 #define CLIP_SW_0_255(in) \ 1083 v4i32 max_m = __msa_ldi_w(255); \ 1086 out_m = __msa_maxi_s_w((v4i32) in, 0); \ 1087 out_m = __msa_min_s_w((v4i32) max_m, (v4i32) out_m); \ 1098 #define HADD_SW_S32(in) \ 1100 v2i64 res0_m, res1_m; \ 1103 res0_m = __msa_hadd_s_d((v4i32) in, (v4i32) in); \ 1104 res1_m = __msa_splati_d(res0_m, 1); \ 1106 sum_m = __msa_copy_s_w((v4i32) res0_m, 0); \ 1117 #define HADD_UH_U32(in) \ 1120 v2u64 res0_m, res1_m; \ 1123 res_m = __msa_hadd_u_w((v8u16) in, (v8u16) in); \ 1124 res0_m = __msa_hadd_u_d(res_m, res_m); \ 1125 res1_m = (v2u64) __msa_splati_d((v2i64) res0_m, 1); \ 1127 sum_m = __msa_copy_u_w((v4i32) res0_m, 0); \ 1139 #define HADD_SB2(RTYPE, in0, in1, out0, out1) \ 1141 out0 = (RTYPE) __msa_hadd_s_h((v16i8) in0, (v16i8) in0); \ 1142 out1 = (RTYPE) __msa_hadd_s_h((v16i8) in1, (v16i8) in1); \ 1144 #define HADD_SB2_SH(...) HADD_SB2(v8i16, __VA_ARGS__) 1146 #define HADD_SB4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) \ 1148 HADD_SB2(RTYPE, in0, in1, out0, out1); \ 1149 HADD_SB2(RTYPE, in2, in3, out2, out3); \ 1151 #define HADD_SB4_UH(...) HADD_SB4(v8u16, __VA_ARGS__) 1152 #define HADD_SB4_SH(...) HADD_SB4(v8i16, __VA_ARGS__) 1162 #define HADD_UB2(RTYPE, in0, in1, out0, out1) \ 1164 out0 = (RTYPE) __msa_hadd_u_h((v16u8) in0, (v16u8) in0); \ 1165 out1 = (RTYPE) __msa_hadd_u_h((v16u8) in1, (v16u8) in1); \ 1167 #define HADD_UB2_UH(...) HADD_UB2(v8u16, __VA_ARGS__) 1169 #define HADD_UB3(RTYPE, in0, in1, in2, out0, out1, out2) \ 1171 HADD_UB2(RTYPE, in0, in1, out0, out1); \ 1172 out2 = (RTYPE) __msa_hadd_u_h((v16u8) in2, (v16u8) in2); \ 1174 #define HADD_UB3_UH(...) HADD_UB3(v8u16, __VA_ARGS__) 1176 #define HADD_UB4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) \ 1178 HADD_UB2(RTYPE, in0, in1, out0, out1); \ 1179 HADD_UB2(RTYPE, in2, in3, out2, out3); \ 1181 #define HADD_UB4_UB(...) HADD_UB4(v16u8, __VA_ARGS__) 1182 #define HADD_UB4_UH(...) HADD_UB4(v8u16, __VA_ARGS__) 1183 #define HADD_UB4_SH(...) HADD_UB4(v8i16, __VA_ARGS__) 1193 #define HSUB_UB2(RTYPE, in0, in1, out0, out1) \ 1195 out0 = (RTYPE) __msa_hsub_u_h((v16u8) in0, (v16u8) in0); \ 1196 out1 = (RTYPE) __msa_hsub_u_h((v16u8) in1, (v16u8) in1); \ 1198 #define HSUB_UB2_UH(...) HSUB_UB2(v8u16, __VA_ARGS__) 1199 #define HSUB_UB2_SH(...) HSUB_UB2(v8i16, __VA_ARGS__) 1201 #define HSUB_UB4(RTYPE, in0, in1, in2, in3, out0, out1, out2, out3) \ 1203 HSUB_UB2(RTYPE, in0, in1, out0, out1); \ 1204 HSUB_UB2(RTYPE, in2, in3, out2, out3); \ 1206 #define HSUB_UB4_UH(...) HSUB_UB4(v8u16, __VA_ARGS__) 1207 #define HSUB_UB4_SH(...) HSUB_UB4(v8i16, __VA_ARGS__) 1218 #define SAD_UB2_UH(in0, in1, ref0, ref1) \ 1220 v16u8 diff0_m, diff1_m; \ 1221 v8u16 sad_m = { 0 }; \ 1223 diff0_m = __msa_asub_u_b((v16u8) in0, (v16u8) ref0); \ 1224 diff1_m = __msa_asub_u_b((v16u8) in1, (v16u8) ref1); \ 1226 sad_m += __msa_hadd_u_h((v16u8) diff0_m, (v16u8) diff0_m); \ 1227 sad_m += __msa_hadd_u_h((v16u8) diff1_m, (v16u8) diff1_m); \ 1238 #define INSERT_W2(RTYPE, in0, in1, out) \ 1240 out = (RTYPE) __msa_insert_w((v4i32) out, 0, in0); \ 1241 out = (RTYPE) __msa_insert_w((v4i32) out, 1, in1); \ 1243 #define INSERT_W2_UB(...) INSERT_W2(v16u8, __VA_ARGS__) 1244 #define INSERT_W2_SB(...) INSERT_W2(v16i8, __VA_ARGS__) 1246 #define INSERT_W4(RTYPE, in0, in1, in2, in3, out) \ 1248 out = (RTYPE) __msa_insert_w((v4i32) out, 0, in0); \ 1249 out = (RTYPE) __msa_insert_w((v4i32) out, 1, in1); \ 1250 out = (RTYPE) __msa_insert_w((v4i32) out, 2, in2); \ 1251 out = (RTYPE) __msa_insert_w((v4i32) out, 3, in3); \ 1253 #define INSERT_W4_UB(...) INSERT_W4(v16u8, __VA_ARGS__) 1254 #define INSERT_W4_SB(...) INSERT_W4(v16i8, __VA_ARGS__) 1255 #define INSERT_W4_SH(...) INSERT_W4(v8i16, __VA_ARGS__) 1256 #define INSERT_W4_SW(...) INSERT_W4(v4i32, __VA_ARGS__) 1264 #define INSERT_D2(RTYPE, in0, in1, out) \ 1266 out = (RTYPE) __msa_insert_d((v2i64) out, 0, in0); \ 1267 out = (RTYPE) __msa_insert_d((v2i64) out, 1, in1); \ 1269 #define INSERT_D2_UB(...) INSERT_D2(v16u8, __VA_ARGS__) 1270 #define INSERT_D2_SB(...) INSERT_D2(v16i8, __VA_ARGS__) 1271 #define INSERT_D2_SH(...) INSERT_D2(v8i16, __VA_ARGS__) 1272 #define INSERT_D2_SD(...) INSERT_D2(v2i64, __VA_ARGS__) 1283 #define ILVEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1285 out0 = (RTYPE) __msa_ilvev_b((v16i8) in1, (v16i8) in0); \ 1286 out1 = (RTYPE) __msa_ilvev_b((v16i8) in3, (v16i8) in2); \ 1288 #define ILVEV_B2_UB(...) ILVEV_B2(v16u8, __VA_ARGS__) 1289 #define ILVEV_B2_SB(...) ILVEV_B2(v16i8, __VA_ARGS__) 1290 #define ILVEV_B2_SH(...) ILVEV_B2(v8i16, __VA_ARGS__) 1291 #define ILVEV_B2_SD(...) ILVEV_B2(v2i64, __VA_ARGS__) 1302 #define ILVEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1304 out0 = (RTYPE) __msa_ilvev_h((v8i16) in1, (v8i16) in0); \ 1305 out1 = (RTYPE) __msa_ilvev_h((v8i16) in3, (v8i16) in2); \ 1307 #define ILVEV_H2_UB(...) ILVEV_H2(v16u8, __VA_ARGS__) 1308 #define ILVEV_H2_SH(...) ILVEV_H2(v8i16, __VA_ARGS__) 1309 #define ILVEV_H2_SW(...) ILVEV_H2(v4i32, __VA_ARGS__) 1320 #define ILVEV_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1322 out0 = (RTYPE) __msa_ilvev_w((v4i32) in1, (v4i32) in0); \ 1323 out1 = (RTYPE) __msa_ilvev_w((v4i32) in3, (v4i32) in2); \ 1325 #define ILVEV_W2_UB(...) ILVEV_W2(v16u8, __VA_ARGS__) 1326 #define ILVEV_W2_SB(...) ILVEV_W2(v16i8, __VA_ARGS__) 1327 #define ILVEV_W2_UH(...) ILVEV_W2(v8u16, __VA_ARGS__) 1328 #define ILVEV_W2_SD(...) ILVEV_W2(v2i64, __VA_ARGS__) 1339 #define ILVEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1341 out0 = (RTYPE) __msa_ilvev_d((v2i64) in1, (v2i64) in0); \ 1342 out1 = (RTYPE) __msa_ilvev_d((v2i64) in3, (v2i64) in2); \ 1344 #define ILVEV_D2_UB(...) ILVEV_D2(v16u8, __VA_ARGS__) 1345 #define ILVEV_D2_SB(...) ILVEV_D2(v16i8, __VA_ARGS__) 1346 #define ILVEV_D2_SW(...) ILVEV_D2(v4i32, __VA_ARGS__) 1357 #define ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1359 out0 = (RTYPE) __msa_ilvl_b((v16i8) in0, (v16i8) in1); \ 1360 out1 = (RTYPE) __msa_ilvl_b((v16i8) in2, (v16i8) in3); \ 1362 #define ILVL_B2_UB(...) ILVL_B2(v16u8, __VA_ARGS__) 1363 #define ILVL_B2_SB(...) ILVL_B2(v16i8, __VA_ARGS__) 1364 #define ILVL_B2_UH(...) ILVL_B2(v8u16, __VA_ARGS__) 1365 #define ILVL_B2_SH(...) ILVL_B2(v8i16, __VA_ARGS__) 1367 #define ILVL_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1368 out0, out1, out2, out3) \ 1370 ILVL_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1371 ILVL_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1373 #define ILVL_B4_UB(...) ILVL_B4(v16u8, __VA_ARGS__) 1374 #define ILVL_B4_SB(...) ILVL_B4(v16i8, __VA_ARGS__) 1375 #define ILVL_B4_UH(...) ILVL_B4(v8u16, __VA_ARGS__) 1376 #define ILVL_B4_SH(...) ILVL_B4(v8i16, __VA_ARGS__) 1387 #define ILVL_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1389 out0 = (RTYPE) __msa_ilvl_h((v8i16) in0, (v8i16) in1); \ 1390 out1 = (RTYPE) __msa_ilvl_h((v8i16) in2, (v8i16) in3); \ 1392 #define ILVL_H2_SH(...) ILVL_H2(v8i16, __VA_ARGS__) 1393 #define ILVL_H2_SW(...) ILVL_H2(v4i32, __VA_ARGS__) 1395 #define ILVL_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1396 out0, out1, out2, out3) \ 1398 ILVL_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1399 ILVL_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1401 #define ILVL_H4_SH(...) ILVL_H4(v8i16, __VA_ARGS__) 1402 #define ILVL_H4_SW(...) ILVL_H4(v4i32, __VA_ARGS__) 1413 #define ILVL_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1415 out0 = (RTYPE) __msa_ilvl_w((v4i32) in0, (v4i32) in1); \ 1416 out1 = (RTYPE) __msa_ilvl_w((v4i32) in2, (v4i32) in3); \ 1418 #define ILVL_W2_UB(...) ILVL_W2(v16u8, __VA_ARGS__) 1419 #define ILVL_W2_SB(...) ILVL_W2(v16i8, __VA_ARGS__) 1420 #define ILVL_W2_SH(...) ILVL_W2(v8i16, __VA_ARGS__) 1432 #define ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1434 out0 = (RTYPE) __msa_ilvr_b((v16i8) in0, (v16i8) in1); \ 1435 out1 = (RTYPE) __msa_ilvr_b((v16i8) in2, (v16i8) in3); \ 1437 #define ILVR_B2_UB(...) ILVR_B2(v16u8, __VA_ARGS__) 1438 #define ILVR_B2_SB(...) ILVR_B2(v16i8, __VA_ARGS__) 1439 #define ILVR_B2_UH(...) ILVR_B2(v8u16, __VA_ARGS__) 1440 #define ILVR_B2_SH(...) ILVR_B2(v8i16, __VA_ARGS__) 1441 #define ILVR_B2_SW(...) ILVR_B2(v4i32, __VA_ARGS__) 1443 #define ILVR_B3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ 1445 ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1446 out2 = (RTYPE) __msa_ilvr_b((v16i8) in4, (v16i8) in5); \ 1448 #define ILVR_B3_UB(...) ILVR_B3(v16u8, __VA_ARGS__) 1449 #define ILVR_B3_SB(...) ILVR_B3(v16i8, __VA_ARGS__) 1450 #define ILVR_B3_UH(...) ILVR_B3(v8u16, __VA_ARGS__) 1451 #define ILVR_B3_SH(...) ILVR_B3(v8i16, __VA_ARGS__) 1453 #define ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1454 out0, out1, out2, out3) \ 1456 ILVR_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1457 ILVR_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1459 #define ILVR_B4_UB(...) ILVR_B4(v16u8, __VA_ARGS__) 1460 #define ILVR_B4_SB(...) ILVR_B4(v16i8, __VA_ARGS__) 1461 #define ILVR_B4_UH(...) ILVR_B4(v8u16, __VA_ARGS__) 1462 #define ILVR_B4_SH(...) ILVR_B4(v8i16, __VA_ARGS__) 1463 #define ILVR_B4_SW(...) ILVR_B4(v4i32, __VA_ARGS__) 1465 #define ILVR_B8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1466 in8, in9, in10, in11, in12, in13, in14, in15, \ 1467 out0, out1, out2, out3, out4, out5, out6, out7) \ 1469 ILVR_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1470 out0, out1, out2, out3); \ 1471 ILVR_B4(RTYPE, in8, in9, in10, in11, in12, in13, in14, in15, \ 1472 out4, out5, out6, out7); \ 1474 #define ILVR_B8_UH(...) ILVR_B8(v8u16, __VA_ARGS__) 1486 #define ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1488 out0 = (RTYPE) __msa_ilvr_h((v8i16) in0, (v8i16) in1); \ 1489 out1 = (RTYPE) __msa_ilvr_h((v8i16) in2, (v8i16) in3); \ 1491 #define ILVR_H2_SH(...) ILVR_H2(v8i16, __VA_ARGS__) 1492 #define ILVR_H2_SW(...) ILVR_H2(v4i32, __VA_ARGS__) 1494 #define ILVR_H3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ 1496 ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1497 out2 = (RTYPE) __msa_ilvr_h((v8i16) in4, (v8i16) in5); \ 1499 #define ILVR_H3_SH(...) ILVR_H3(v8i16, __VA_ARGS__) 1501 #define ILVR_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1502 out0, out1, out2, out3) \ 1504 ILVR_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1505 ILVR_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1507 #define ILVR_H4_SH(...) ILVR_H4(v8i16, __VA_ARGS__) 1508 #define ILVR_H4_SW(...) ILVR_H4(v4i32, __VA_ARGS__) 1510 #define ILVR_W2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1512 out0 = (RTYPE) __msa_ilvr_w((v4i32) in0, (v4i32) in1); \ 1513 out1 = (RTYPE) __msa_ilvr_w((v4i32) in2, (v4i32) in3); \ 1515 #define ILVR_W2_UB(...) ILVR_W2(v16u8, __VA_ARGS__) 1516 #define ILVR_W2_SB(...) ILVR_W2(v16i8, __VA_ARGS__) 1517 #define ILVR_W2_SH(...) ILVR_W2(v8i16, __VA_ARGS__) 1519 #define ILVR_W4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1520 out0, out1, out2, out3) \ 1522 ILVR_W2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1523 ILVR_W2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1525 #define ILVR_W4_SB(...) ILVR_W4(v16i8, __VA_ARGS__) 1526 #define ILVR_W4_UB(...) ILVR_W4(v16u8, __VA_ARGS__) 1537 #define ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1539 out0 = (RTYPE) __msa_ilvr_d((v2i64) in0, (v2i64) in1); \ 1540 out1 = (RTYPE) __msa_ilvr_d((v2i64) in2, (v2i64) in3); \ 1542 #define ILVR_D2_UB(...) ILVR_D2(v16u8, __VA_ARGS__) 1543 #define ILVR_D2_SB(...) ILVR_D2(v16i8, __VA_ARGS__) 1544 #define ILVR_D2_SH(...) ILVR_D2(v8i16, __VA_ARGS__) 1546 #define ILVR_D3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ 1548 ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1549 out2 = (RTYPE) __msa_ilvr_d((v2i64) in4, (v2i64) in5); \ 1551 #define ILVR_D3_SB(...) ILVR_D3(v16i8, __VA_ARGS__) 1553 #define ILVR_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1554 out0, out1, out2, out3) \ 1556 ILVR_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1557 ILVR_D2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1559 #define ILVR_D4_SB(...) ILVR_D4(v16i8, __VA_ARGS__) 1560 #define ILVR_D4_UB(...) ILVR_D4(v16u8, __VA_ARGS__) 1571 #define ILVL_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1573 out0 = (RTYPE) __msa_ilvl_d((v2i64) in0, (v2i64) in1); \ 1574 out1 = (RTYPE) __msa_ilvl_d((v2i64) in2, (v2i64) in3); \ 1576 #define ILVL_D2_UB(...) ILVL_D2(v16u8, __VA_ARGS__) 1577 #define ILVL_D2_SB(...) ILVL_D2(v16i8, __VA_ARGS__) 1578 #define ILVL_D2_SH(...) ILVL_D2(v8i16, __VA_ARGS__) 1589 #define ILVRL_B2(RTYPE, in0, in1, out0, out1) \ 1591 out0 = (RTYPE) __msa_ilvr_b((v16i8) in0, (v16i8) in1); \ 1592 out1 = (RTYPE) __msa_ilvl_b((v16i8) in0, (v16i8) in1); \ 1594 #define ILVRL_B2_UB(...) ILVRL_B2(v16u8, __VA_ARGS__) 1595 #define ILVRL_B2_SB(...) ILVRL_B2(v16i8, __VA_ARGS__) 1596 #define ILVRL_B2_UH(...) ILVRL_B2(v8u16, __VA_ARGS__) 1597 #define ILVRL_B2_SH(...) ILVRL_B2(v8i16, __VA_ARGS__) 1598 #define ILVRL_B2_SW(...) ILVRL_B2(v4i32, __VA_ARGS__) 1600 #define ILVRL_H2(RTYPE, in0, in1, out0, out1) \ 1602 out0 = (RTYPE) __msa_ilvr_h((v8i16) in0, (v8i16) in1); \ 1603 out1 = (RTYPE) __msa_ilvl_h((v8i16) in0, (v8i16) in1); \ 1605 #define ILVRL_H2_UB(...) ILVRL_H2(v16u8, __VA_ARGS__) 1606 #define ILVRL_H2_SB(...) ILVRL_H2(v16i8, __VA_ARGS__) 1607 #define ILVRL_H2_SH(...) ILVRL_H2(v8i16, __VA_ARGS__) 1608 #define ILVRL_H2_SW(...) ILVRL_H2(v4i32, __VA_ARGS__) 1610 #define ILVRL_W2(RTYPE, in0, in1, out0, out1) \ 1612 out0 = (RTYPE) __msa_ilvr_w((v4i32) in0, (v4i32) in1); \ 1613 out1 = (RTYPE) __msa_ilvl_w((v4i32) in0, (v4i32) in1); \ 1615 #define ILVRL_W2_UB(...) ILVRL_W2(v16u8, __VA_ARGS__) 1616 #define ILVRL_W2_SH(...) ILVRL_W2(v8i16, __VA_ARGS__) 1617 #define ILVRL_W2_SW(...) ILVRL_W2(v4i32, __VA_ARGS__) 1627 #define MAXI_SH2(RTYPE, in0, in1, max_val) \ 1629 in0 = (RTYPE) __msa_maxi_s_h((v8i16) in0, max_val); \ 1630 in1 = (RTYPE) __msa_maxi_s_h((v8i16) in1, max_val); \ 1632 #define MAXI_SH2_UH(...) MAXI_SH2(v8u16, __VA_ARGS__) 1633 #define MAXI_SH2_SH(...) MAXI_SH2(v8i16, __VA_ARGS__) 1635 #define MAXI_SH4(RTYPE, in0, in1, in2, in3, max_val) \ 1637 MAXI_SH2(RTYPE, in0, in1, max_val); \ 1638 MAXI_SH2(RTYPE, in2, in3, max_val); \ 1640 #define MAXI_SH4_UH(...) MAXI_SH4(v8u16, __VA_ARGS__) 1641 #define MAXI_SH4_SH(...) MAXI_SH4(v8i16, __VA_ARGS__) 1643 #define MAXI_SH8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, max_val) \ 1645 MAXI_SH4(RTYPE, in0, in1, in2, in3, max_val); \ 1646 MAXI_SH4(RTYPE, in4, in5, in6, in7, max_val); \ 1648 #define MAXI_SH8_UH(...) MAXI_SH8(v8u16, __VA_ARGS__) 1649 #define MAXI_SH8_SH(...) MAXI_SH8(v8i16, __VA_ARGS__) 1661 #define SAT_UH2(RTYPE, in0, in1, sat_val) \ 1663 in0 = (RTYPE) __msa_sat_u_h((v8u16) in0, sat_val); \ 1664 in1 = (RTYPE) __msa_sat_u_h((v8u16) in1, sat_val); \ 1666 #define SAT_UH2_UH(...) SAT_UH2(v8u16, __VA_ARGS__) 1667 #define SAT_UH2_SH(...) SAT_UH2(v8i16, __VA_ARGS__) 1669 #define SAT_UH4(RTYPE, in0, in1, in2, in3, sat_val) \ 1671 SAT_UH2(RTYPE, in0, in1, sat_val); \ 1672 SAT_UH2(RTYPE, in2, in3, sat_val); \ 1674 #define SAT_UH4_UH(...) SAT_UH4(v8u16, __VA_ARGS__) 1675 #define SAT_UH4_SH(...) SAT_UH4(v8i16, __VA_ARGS__) 1677 #define SAT_UH8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, sat_val) \ 1679 SAT_UH4(RTYPE, in0, in1, in2, in3, sat_val); \ 1680 SAT_UH4(RTYPE, in4, in5, in6, in7, sat_val); \ 1682 #define SAT_UH8_UH(...) SAT_UH8(v8u16, __VA_ARGS__) 1683 #define SAT_UH8_SH(...) SAT_UH8(v8i16, __VA_ARGS__) 1695 #define SAT_SH2(RTYPE, in0, in1, sat_val) \ 1697 in0 = (RTYPE) __msa_sat_s_h((v8i16) in0, sat_val); \ 1698 in1 = (RTYPE) __msa_sat_s_h((v8i16) in1, sat_val); \ 1700 #define SAT_SH2_SH(...) SAT_SH2(v8i16, __VA_ARGS__) 1702 #define SAT_SH3(RTYPE, in0, in1, in2, sat_val) \ 1704 SAT_SH2(RTYPE, in0, in1, sat_val); \ 1705 in2 = (RTYPE) __msa_sat_s_h((v8i16) in2, sat_val); \ 1707 #define SAT_SH3_SH(...) SAT_SH3(v8i16, __VA_ARGS__) 1709 #define SAT_SH4(RTYPE, in0, in1, in2, in3, sat_val) \ 1711 SAT_SH2(RTYPE, in0, in1, sat_val); \ 1712 SAT_SH2(RTYPE, in2, in3, sat_val); \ 1714 #define SAT_SH4_SH(...) SAT_SH4(v8i16, __VA_ARGS__) 1726 #define SAT_SW2(RTYPE, in0, in1, sat_val) \ 1728 in0 = (RTYPE) __msa_sat_s_w((v4i32) in0, sat_val); \ 1729 in1 = (RTYPE) __msa_sat_s_w((v4i32) in1, sat_val); \ 1731 #define SAT_SW2_SW(...) SAT_SW2(v4i32, __VA_ARGS__) 1733 #define SAT_SW4(RTYPE, in0, in1, in2, in3, sat_val) \ 1735 SAT_SW2(RTYPE, in0, in1, sat_val); \ 1736 SAT_SW2(RTYPE, in2, in3, sat_val); \ 1738 #define SAT_SW4_SW(...) SAT_SW4(v4i32, __VA_ARGS__) 1749 #define SPLATI_H2(RTYPE, in, idx0, idx1, out0, out1) \ 1751 out0 = (RTYPE) __msa_splati_h((v8i16) in, idx0); \ 1752 out1 = (RTYPE) __msa_splati_h((v8i16) in, idx1); \ 1754 #define SPLATI_H2_SB(...) SPLATI_H2(v16i8, __VA_ARGS__) 1755 #define SPLATI_H2_SH(...) SPLATI_H2(v8i16, __VA_ARGS__) 1757 #define SPLATI_H3(RTYPE, in, idx0, idx1, idx2, \ 1760 SPLATI_H2(RTYPE, in, idx0, idx1, out0, out1); \ 1761 out2 = (RTYPE) __msa_splati_h((v8i16) in, idx2); \ 1763 #define SPLATI_H3_SB(...) SPLATI_H3(v16i8, __VA_ARGS__) 1764 #define SPLATI_H3_SH(...) SPLATI_H3(v8i16, __VA_ARGS__) 1766 #define SPLATI_H4(RTYPE, in, idx0, idx1, idx2, idx3, \ 1767 out0, out1, out2, out3) \ 1769 SPLATI_H2(RTYPE, in, idx0, idx1, out0, out1); \ 1770 SPLATI_H2(RTYPE, in, idx2, idx3, out2, out3); \ 1772 #define SPLATI_H4_SB(...) SPLATI_H4(v16i8, __VA_ARGS__) 1773 #define SPLATI_H4_SH(...) SPLATI_H4(v8i16, __VA_ARGS__) 1786 #define SPLATI_W2(RTYPE, in, stidx, out0, out1) \ 1788 out0 = (RTYPE) __msa_splati_w((v4i32) in, stidx); \ 1789 out1 = (RTYPE) __msa_splati_w((v4i32) in, (stidx+1)); \ 1791 #define SPLATI_W2_SH(...) SPLATI_W2(v8i16, __VA_ARGS__) 1792 #define SPLATI_W2_SW(...) SPLATI_W2(v4i32, __VA_ARGS__) 1794 #define SPLATI_W4(RTYPE, in, out0, out1, out2, out3) \ 1796 SPLATI_W2(RTYPE, in, 0, out0, out1); \ 1797 SPLATI_W2(RTYPE, in, 2, out2, out3); \ 1799 #define SPLATI_W4_SH(...) SPLATI_W4(v8i16, __VA_ARGS__) 1800 #define SPLATI_W4_SW(...) SPLATI_W4(v4i32, __VA_ARGS__) 1813 #define PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1815 out0 = (RTYPE) __msa_pckev_b((v16i8) in0, (v16i8) in1); \ 1816 out1 = (RTYPE) __msa_pckev_b((v16i8) in2, (v16i8) in3); \ 1818 #define PCKEV_B2_SB(...) PCKEV_B2(v16i8, __VA_ARGS__) 1819 #define PCKEV_B2_UB(...) PCKEV_B2(v16u8, __VA_ARGS__) 1820 #define PCKEV_B2_SH(...) PCKEV_B2(v8i16, __VA_ARGS__) 1821 #define PCKEV_B2_SW(...) PCKEV_B2(v4i32, __VA_ARGS__) 1823 #define PCKEV_B3(RTYPE, in0, in1, in2, in3, in4, in5, out0, out1, out2) \ 1825 PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1826 out2 = (RTYPE) __msa_pckev_b((v16i8) in4, (v16i8) in5); \ 1828 #define PCKEV_B3_UB(...) PCKEV_B3(v16u8, __VA_ARGS__) 1829 #define PCKEV_B3_SB(...) PCKEV_B3(v16i8, __VA_ARGS__) 1831 #define PCKEV_B4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1832 out0, out1, out2, out3) \ 1834 PCKEV_B2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1835 PCKEV_B2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1837 #define PCKEV_B4_SB(...) PCKEV_B4(v16i8, __VA_ARGS__) 1838 #define PCKEV_B4_UB(...) PCKEV_B4(v16u8, __VA_ARGS__) 1839 #define PCKEV_B4_SH(...) PCKEV_B4(v8i16, __VA_ARGS__) 1840 #define PCKEV_B4_SW(...) PCKEV_B4(v4i32, __VA_ARGS__) 1853 #define PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1855 out0 = (RTYPE) __msa_pckev_h((v8i16) in0, (v8i16) in1); \ 1856 out1 = (RTYPE) __msa_pckev_h((v8i16) in2, (v8i16) in3); \ 1858 #define PCKEV_H2_SH(...) PCKEV_H2(v8i16, __VA_ARGS__) 1859 #define PCKEV_H2_SW(...) PCKEV_H2(v4i32, __VA_ARGS__) 1861 #define PCKEV_H4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1862 out0, out1, out2, out3) \ 1864 PCKEV_H2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1865 PCKEV_H2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1867 #define PCKEV_H4_SH(...) PCKEV_H4(v8i16, __VA_ARGS__) 1868 #define PCKEV_H4_SW(...) PCKEV_H4(v4i32, __VA_ARGS__) 1881 #define PCKEV_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1883 out0 = (RTYPE) __msa_pckev_d((v2i64) in0, (v2i64) in1); \ 1884 out1 = (RTYPE) __msa_pckev_d((v2i64) in2, (v2i64) in3); \ 1886 #define PCKEV_D2_UB(...) PCKEV_D2(v16u8, __VA_ARGS__) 1887 #define PCKEV_D2_SB(...) PCKEV_D2(v16i8, __VA_ARGS__) 1888 #define PCKEV_D2_SH(...) PCKEV_D2(v8i16, __VA_ARGS__) 1890 #define PCKEV_D4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1891 out0, out1, out2, out3) \ 1893 PCKEV_D2(RTYPE, in0, in1, in2, in3, out0, out1); \ 1894 PCKEV_D2(RTYPE, in4, in5, in6, in7, out2, out3); \ 1896 #define PCKEV_D4_UB(...) PCKEV_D4(v16u8, __VA_ARGS__) 1907 #define PCKOD_D2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1909 out0 = (RTYPE) __msa_pckod_d((v2i64) in0, (v2i64) in1); \ 1910 out1 = (RTYPE) __msa_pckod_d((v2i64) in2, (v2i64) in3); \ 1912 #define PCKOD_D2_UB(...) PCKOD_D2(v16u8, __VA_ARGS__) 1913 #define PCKOD_D2_SH(...) PCKOD_D2(v8i16, __VA_ARGS__) 1914 #define PCKOD_D2_SD(...) PCKOD_D2(v2i64, __VA_ARGS__) 1928 #define XORI_B2_128(RTYPE, in0, in1) \ 1930 in0 = (RTYPE) __msa_xori_b((v16u8) in0, 128); \ 1931 in1 = (RTYPE) __msa_xori_b((v16u8) in1, 128); \ 1933 #define XORI_B2_128_UB(...) XORI_B2_128(v16u8, __VA_ARGS__) 1934 #define XORI_B2_128_SB(...) XORI_B2_128(v16i8, __VA_ARGS__) 1935 #define XORI_B2_128_SH(...) XORI_B2_128(v8i16, __VA_ARGS__) 1937 #define XORI_B3_128(RTYPE, in0, in1, in2) \ 1939 XORI_B2_128(RTYPE, in0, in1); \ 1940 in2 = (RTYPE) __msa_xori_b((v16u8) in2, 128); \ 1942 #define XORI_B3_128_SB(...) XORI_B3_128(v16i8, __VA_ARGS__) 1944 #define XORI_B4_128(RTYPE, in0, in1, in2, in3) \ 1946 XORI_B2_128(RTYPE, in0, in1); \ 1947 XORI_B2_128(RTYPE, in2, in3); \ 1949 #define XORI_B4_128_UB(...) XORI_B4_128(v16u8, __VA_ARGS__) 1950 #define XORI_B4_128_SB(...) XORI_B4_128(v16i8, __VA_ARGS__) 1951 #define XORI_B4_128_SH(...) XORI_B4_128(v8i16, __VA_ARGS__) 1953 #define XORI_B5_128(RTYPE, in0, in1, in2, in3, in4) \ 1955 XORI_B3_128(RTYPE, in0, in1, in2); \ 1956 XORI_B2_128(RTYPE, in3, in4); \ 1958 #define XORI_B5_128_SB(...) XORI_B5_128(v16i8, __VA_ARGS__) 1960 #define XORI_B6_128(RTYPE, in0, in1, in2, in3, in4, in5) \ 1962 XORI_B4_128(RTYPE, in0, in1, in2, in3); \ 1963 XORI_B2_128(RTYPE, in4, in5); \ 1965 #define XORI_B6_128_SB(...) XORI_B6_128(v16i8, __VA_ARGS__) 1967 #define XORI_B7_128(RTYPE, in0, in1, in2, in3, in4, in5, in6) \ 1969 XORI_B4_128(RTYPE, in0, in1, in2, in3); \ 1970 XORI_B3_128(RTYPE, in4, in5, in6); \ 1972 #define XORI_B7_128_SB(...) XORI_B7_128(v16i8, __VA_ARGS__) 1974 #define XORI_B8_128(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7) \ 1976 XORI_B4_128(RTYPE, in0, in1, in2, in3); \ 1977 XORI_B4_128(RTYPE, in4, in5, in6, in7); \ 1979 #define XORI_B8_128_SB(...) XORI_B8_128(v16i8, __VA_ARGS__) 1980 #define XORI_B8_128_UB(...) XORI_B8_128(v16u8, __VA_ARGS__) 1991 #define ADDS_SH2(RTYPE, in0, in1, in2, in3, out0, out1) \ 1993 out0 = (RTYPE) __msa_adds_s_h((v8i16) in0, (v8i16) in1); \ 1994 out1 = (RTYPE) __msa_adds_s_h((v8i16) in2, (v8i16) in3); \ 1996 #define ADDS_SH2_SH(...) ADDS_SH2(v8i16, __VA_ARGS__) 1998 #define ADDS_SH4(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 1999 out0, out1, out2, out3) \ 2001 ADDS_SH2(RTYPE, in0, in1, in2, in3, out0, out1); \ 2002 ADDS_SH2(RTYPE, in4, in5, in6, in7, out2, out3); \ 2004 #define ADDS_SH4_UH(...) ADDS_SH4(v8u16, __VA_ARGS__) 2005 #define ADDS_SH4_SH(...) ADDS_SH4(v8i16, __VA_ARGS__) 2015 #define SLLI_2V(in0, in1, shift) \ 2017 in0 = in0 << shift; \ 2018 in1 = in1 << shift; \ 2020 #define SLLI_4V(in0, in1, in2, in3, shift) \ 2022 in0 = in0 << shift; \ 2023 in1 = in1 << shift; \ 2024 in2 = in2 << shift; \ 2025 in3 = in3 << shift; \ 2038 #define SRA_4V(in0, in1, in2, in3, shift) \ 2040 in0 = in0 >> shift; \ 2041 in1 = in1 >> shift; \ 2042 in2 = in2 >> shift; \ 2043 in3 = in3 >> shift; \ 2056 #define SRL_H4(RTYPE, in0, in1, in2, in3, shift) \ 2058 in0 = (RTYPE) __msa_srl_h((v8i16) in0, (v8i16) shift); \ 2059 in1 = (RTYPE) __msa_srl_h((v8i16) in1, (v8i16) shift); \ 2060 in2 = (RTYPE) __msa_srl_h((v8i16) in2, (v8i16) shift); \ 2061 in3 = (RTYPE) __msa_srl_h((v8i16) in3, (v8i16) shift); \ 2063 #define SRL_H4_UH(...) SRL_H4(v8u16, __VA_ARGS__) 2065 #define SRLR_H4(RTYPE, in0, in1, in2, in3, shift) \ 2067 in0 = (RTYPE) __msa_srlr_h((v8i16) in0, (v8i16) shift); \ 2068 in1 = (RTYPE) __msa_srlr_h((v8i16) in1, (v8i16) shift); \ 2069 in2 = (RTYPE) __msa_srlr_h((v8i16) in2, (v8i16) shift); \ 2070 in3 = (RTYPE) __msa_srlr_h((v8i16) in3, (v8i16) shift); \ 2072 #define SRLR_H4_UH(...) SRLR_H4(v8u16, __VA_ARGS__) 2073 #define SRLR_H4_SH(...) SRLR_H4(v8i16, __VA_ARGS__) 2075 #define SRLR_H8(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, shift) \ 2077 SRLR_H4(RTYPE, in0, in1, in2, in3, shift); \ 2078 SRLR_H4(RTYPE, in4, in5, in6, in7, shift); \ 2080 #define SRLR_H8_UH(...) SRLR_H8(v8u16, __VA_ARGS__) 2081 #define SRLR_H8_SH(...) SRLR_H8(v8i16, __VA_ARGS__) 2094 #define SRAR_H2(RTYPE, in0, in1, shift) \ 2096 in0 = (RTYPE) __msa_srar_h((v8i16) in0, (v8i16) shift); \ 2097 in1 = (RTYPE) __msa_srar_h((v8i16) in1, (v8i16) shift); \ 2099 #define SRAR_H2_UH(...) SRAR_H2(v8u16, __VA_ARGS__) 2100 #define SRAR_H2_SH(...) SRAR_H2(v8i16, __VA_ARGS__) 2102 #define SRAR_H3(RTYPE, in0, in1, in2, shift) \ 2104 SRAR_H2(RTYPE, in0, in1, shift) \ 2105 in2 = (RTYPE) __msa_srar_h((v8i16) in2, (v8i16) shift); \ 2107 #define SRAR_H3_SH(...) SRAR_H3(v8i16, __VA_ARGS__) 2109 #define SRAR_H4(RTYPE, in0, in1, in2, in3, shift) \ 2111 SRAR_H2(RTYPE, in0, in1, shift) \ 2112 SRAR_H2(RTYPE, in2, in3, shift) \ 2114 #define SRAR_H4_UH(...) SRAR_H4(v8u16, __VA_ARGS__) 2115 #define SRAR_H4_SH(...) SRAR_H4(v8i16, __VA_ARGS__) 2128 #define SRAR_W2(RTYPE, in0, in1, shift) \ 2130 in0 = (RTYPE) __msa_srar_w((v4i32) in0, (v4i32) shift); \ 2131 in1 = (RTYPE) __msa_srar_w((v4i32) in1, (v4i32) shift); \ 2133 #define SRAR_W2_SW(...) SRAR_W2(v4i32, __VA_ARGS__) 2135 #define SRAR_W4(RTYPE, in0, in1, in2, in3, shift) \ 2137 SRAR_W2(RTYPE, in0, in1, shift) \ 2138 SRAR_W2(RTYPE, in2, in3, shift) \ 2140 #define SRAR_W4_SW(...) SRAR_W4(v4i32, __VA_ARGS__) 2152 #define SRARI_H2(RTYPE, in0, in1, shift) \ 2154 in0 = (RTYPE) __msa_srari_h((v8i16) in0, shift); \ 2155 in1 = (RTYPE) __msa_srari_h((v8i16) in1, shift); \ 2157 #define SRARI_H2_UH(...) SRARI_H2(v8u16, __VA_ARGS__) 2158 #define SRARI_H2_SH(...) SRARI_H2(v8i16, __VA_ARGS__) 2160 #define SRARI_H4(RTYPE, in0, in1, in2, in3, shift) \ 2162 SRARI_H2(RTYPE, in0, in1, shift); \ 2163 SRARI_H2(RTYPE, in2, in3, shift); \ 2165 #define SRARI_H4_UH(...) SRARI_H4(v8u16, __VA_ARGS__) 2166 #define SRARI_H4_SH(...) SRARI_H4(v8i16, __VA_ARGS__) 2178 #define SRARI_W2(RTYPE, in0, in1, shift) \ 2180 in0 = (RTYPE) __msa_srari_w((v4i32) in0, shift); \ 2181 in1 = (RTYPE) __msa_srari_w((v4i32) in1, shift); \ 2183 #define SRARI_W2_SW(...) SRARI_W2(v4i32, __VA_ARGS__) 2185 #define SRARI_W4(RTYPE, in0, in1, in2, in3, shift) \ 2187 SRARI_W2(RTYPE, in0, in1, shift); \ 2188 SRARI_W2(RTYPE, in2, in3, shift); \ 2190 #define SRARI_W4_SH(...) SRARI_W4(v8i16, __VA_ARGS__) 2191 #define SRARI_W4_SW(...) SRARI_W4(v4i32, __VA_ARGS__) 2200 #define MUL2(in0, in1, in2, in3, out0, out1) \ 2205 #define MUL4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ 2207 MUL2(in0, in1, in2, in3, out0, out1); \ 2208 MUL2(in4, in5, in6, in7, out2, out3); \ 2217 #define ADD2(in0, in1, in2, in3, out0, out1) \ 2222 #define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ 2224 ADD2(in0, in1, in2, in3, out0, out1); \ 2225 ADD2(in4, in5, in6, in7, out2, out3); \ 2234 #define SUB2(in0, in1, in2, in3, out0, out1) \ 2239 #define SUB4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3) \ 2255 #define UNPCK_R_SH_SW(in, out) \ 2259 sign_m = __msa_clti_s_h((v8i16) in, 0); \ 2260 out = (v4i32) __msa_ilvr_h(sign_m, (v8i16) in); \ 2274 #define UNPCK_SB_SH(in, out0, out1) \ 2278 tmp_m = __msa_clti_s_b((v16i8) in, 0); \ 2279 ILVRL_B2_SH(tmp_m, in, out0, out1); \ 2289 #define UNPCK_UB_SH(in, out0, out1) \ 2291 v16i8 zero_m = { 0 }; \ 2293 ILVRL_B2_SH(zero_m, in, out0, out1); \ 2307 #define UNPCK_SH_SW(in, out0, out1) \ 2311 tmp_m = __msa_clti_s_h((v8i16) in, 0); \ 2312 ILVRL_H2_SW(tmp_m, in, out0, out1); \ 2320 #define SWAP(in0, in1) \ 2332 #define BUTTERFLY_4(in0, in1, in2, in3, out0, out1, out2, out3) \ 2346 #define BUTTERFLY_8(in0, in1, in2, in3, in4, in5, in6, in7, \ 2347 out0, out1, out2, out3, out4, out5, out6, out7) \ 2365 #define BUTTERFLY_16(in0, in1, in2, in3, in4, in5, in6, in7, \ 2366 in8, in9, in10, in11, in12, in13, in14, in15, \ 2367 out0, out1, out2, out3, out4, out5, out6, out7, \ 2368 out8, out9, out10, out11, out12, out13, out14, out15) \ 2370 out0 = in0 + in15; \ 2371 out1 = in1 + in14; \ 2372 out2 = in2 + in13; \ 2373 out3 = in3 + in12; \ 2374 out4 = in4 + in11; \ 2375 out5 = in5 + in10; \ 2381 out10 = in5 - in10; \ 2382 out11 = in4 - in11; \ 2383 out12 = in3 - in12; \ 2384 out13 = in2 - in13; \ 2385 out14 = in1 - in14; \ 2386 out15 = in0 - in15; \ 2395 #define TRANSPOSE4x4_UB_UB(in0, in1, in2, in3, out0, out1, out2, out3) \ 2397 v16i8 zero_m = { 0 }; \ 2398 v16i8 s0_m, s1_m, s2_m, s3_m; \ 2400 ILVR_D2_SB(in1, in0, in3, in2, s0_m, s1_m); \ 2401 ILVRL_B2_SB(s1_m, s0_m, s2_m, s3_m); \ 2403 out0 = (v16u8) __msa_ilvr_b(s3_m, s2_m); \ 2404 out1 = (v16u8) __msa_sldi_b(zero_m, (v16i8) out0, 4); \ 2405 out2 = (v16u8) __msa_sldi_b(zero_m, (v16i8) out1, 4); \ 2406 out3 = (v16u8) __msa_sldi_b(zero_m, (v16i8) out2, 4); \ 2415 #define TRANSPOSE8x4_UB(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 2416 out0, out1, out2, out3) \ 2418 v16i8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2420 ILVEV_W2_SB(in0, in4, in1, in5, tmp0_m, tmp1_m); \ 2421 tmp2_m = __msa_ilvr_b(tmp1_m, tmp0_m); \ 2422 ILVEV_W2_SB(in2, in6, in3, in7, tmp0_m, tmp1_m); \ 2424 tmp3_m = __msa_ilvr_b(tmp1_m, tmp0_m); \ 2425 ILVRL_H2_SB(tmp3_m, tmp2_m, tmp0_m, tmp1_m); \ 2427 ILVRL_W2(RTYPE, tmp1_m, tmp0_m, out0, out2); \ 2428 out1 = (RTYPE) __msa_ilvl_d((v2i64) out2, (v2i64) out0); \ 2429 out3 = (RTYPE) __msa_ilvl_d((v2i64) out0, (v2i64) out2); \ 2431 #define TRANSPOSE8x4_UB_UB(...) TRANSPOSE8x4_UB(v16u8, __VA_ARGS__) 2432 #define TRANSPOSE8x4_UB_UH(...) TRANSPOSE8x4_UB(v8u16, __VA_ARGS__) 2442 #define TRANSPOSE8x8_UB(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 2443 out0, out1, out2, out3, out4, out5, out6, out7) \ 2445 v16i8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2446 v16i8 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ 2448 ILVR_B4_SB(in2, in0, in3, in1, in6, in4, in7, in5, \ 2449 tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ 2450 ILVRL_B2_SB(tmp1_m, tmp0_m, tmp4_m, tmp5_m); \ 2451 ILVRL_B2_SB(tmp3_m, tmp2_m, tmp6_m, tmp7_m); \ 2452 ILVRL_W2(RTYPE, tmp6_m, tmp4_m, out0, out2); \ 2453 ILVRL_W2(RTYPE, tmp7_m, tmp5_m, out4, out6); \ 2454 SLDI_B2_0(RTYPE, out0, out2, out1, out3, 8); \ 2455 SLDI_B2_0(RTYPE, out4, out6, out5, out7, 8); \ 2457 #define TRANSPOSE8x8_UB_UB(...) TRANSPOSE8x8_UB(v16u8, __VA_ARGS__) 2458 #define TRANSPOSE8x8_UB_UH(...) TRANSPOSE8x8_UB(v8u16, __VA_ARGS__) 2467 #define TRANSPOSE16x4_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2468 in8, in9, in10, in11, in12, in13, in14, in15, \ 2469 out0, out1, out2, out3) \ 2471 v2i64 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2473 ILVEV_W2_SD(in0, in4, in8, in12, tmp0_m, tmp1_m); \ 2474 out1 = (v16u8) __msa_ilvev_d(tmp1_m, tmp0_m); \ 2476 ILVEV_W2_SD(in1, in5, in9, in13, tmp0_m, tmp1_m); \ 2477 out3 = (v16u8) __msa_ilvev_d(tmp1_m, tmp0_m); \ 2479 ILVEV_W2_SD(in2, in6, in10, in14, tmp0_m, tmp1_m); \ 2481 tmp2_m = __msa_ilvev_d(tmp1_m, tmp0_m); \ 2482 ILVEV_W2_SD(in3, in7, in11, in15, tmp0_m, tmp1_m); \ 2484 tmp3_m = __msa_ilvev_d(tmp1_m, tmp0_m); \ 2485 ILVEV_B2_SD(out1, out3, tmp2_m, tmp3_m, tmp0_m, tmp1_m); \ 2486 out0 = (v16u8) __msa_ilvev_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2487 out2 = (v16u8) __msa_ilvod_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2489 tmp0_m = (v2i64) __msa_ilvod_b((v16i8) out3, (v16i8) out1); \ 2490 tmp1_m = (v2i64) __msa_ilvod_b((v16i8) tmp3_m, (v16i8) tmp2_m); \ 2491 out1 = (v16u8) __msa_ilvev_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2492 out3 = (v16u8) __msa_ilvod_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2502 #define TRANSPOSE16x8_UB_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2503 in8, in9, in10, in11, in12, in13, in14, in15, \ 2504 out0, out1, out2, out3, out4, out5, out6, out7) \ 2506 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2507 v16u8 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ 2509 ILVEV_D2_UB(in0, in8, in1, in9, out7, out6); \ 2510 ILVEV_D2_UB(in2, in10, in3, in11, out5, out4); \ 2511 ILVEV_D2_UB(in4, in12, in5, in13, out3, out2); \ 2512 ILVEV_D2_UB(in6, in14, in7, in15, out1, out0); \ 2514 tmp0_m = (v16u8) __msa_ilvev_b((v16i8) out6, (v16i8) out7); \ 2515 tmp4_m = (v16u8) __msa_ilvod_b((v16i8) out6, (v16i8) out7); \ 2516 tmp1_m = (v16u8) __msa_ilvev_b((v16i8) out4, (v16i8) out5); \ 2517 tmp5_m = (v16u8) __msa_ilvod_b((v16i8) out4, (v16i8) out5); \ 2518 out5 = (v16u8) __msa_ilvev_b((v16i8) out2, (v16i8) out3); \ 2519 tmp6_m = (v16u8) __msa_ilvod_b((v16i8) out2, (v16i8) out3); \ 2520 out7 = (v16u8) __msa_ilvev_b((v16i8) out0, (v16i8) out1); \ 2521 tmp7_m = (v16u8) __msa_ilvod_b((v16i8) out0, (v16i8) out1); \ 2523 ILVEV_H2_UB(tmp0_m, tmp1_m, out5, out7, tmp2_m, tmp3_m); \ 2524 out0 = (v16u8) __msa_ilvev_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2525 out4 = (v16u8) __msa_ilvod_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2527 tmp2_m = (v16u8) __msa_ilvod_h((v8i16) tmp1_m, (v8i16) tmp0_m); \ 2528 tmp3_m = (v16u8) __msa_ilvod_h((v8i16) out7, (v8i16) out5); \ 2529 out2 = (v16u8) __msa_ilvev_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2530 out6 = (v16u8) __msa_ilvod_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2532 ILVEV_H2_UB(tmp4_m, tmp5_m, tmp6_m, tmp7_m, tmp2_m, tmp3_m); \ 2533 out1 = (v16u8) __msa_ilvev_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2534 out5 = (v16u8) __msa_ilvod_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2536 tmp2_m = (v16u8) __msa_ilvod_h((v8i16) tmp5_m, (v8i16) tmp4_m); \ 2537 tmp2_m = (v16u8) __msa_ilvod_h((v8i16) tmp5_m, (v8i16) tmp4_m); \ 2538 tmp3_m = (v16u8) __msa_ilvod_h((v8i16) tmp7_m, (v8i16) tmp6_m); \ 2539 tmp3_m = (v16u8) __msa_ilvod_h((v8i16) tmp7_m, (v8i16) tmp6_m); \ 2540 out3 = (v16u8) __msa_ilvev_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2541 out7 = (v16u8) __msa_ilvod_w((v4i32) tmp3_m, (v4i32) tmp2_m); \ 2550 #define TRANSPOSE4x4_SH_SH(in0, in1, in2, in3, out0, out1, out2, out3) \ 2554 ILVR_H2_SH(in1, in0, in3, in2, s0_m, s1_m); \ 2555 ILVRL_W2_SH(s1_m, s0_m, out0, out2); \ 2556 out1 = (v8i16) __msa_ilvl_d((v2i64) out0, (v2i64) out0); \ 2557 out3 = (v8i16) __msa_ilvl_d((v2i64) out0, (v2i64) out2); \ 2566 #define TRANSPOSE8x8_H(RTYPE, in0, in1, in2, in3, in4, in5, in6, in7, \ 2567 out0, out1, out2, out3, out4, out5, out6, out7) \ 2570 v8i16 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2571 v8i16 tmp4_m, tmp5_m, tmp6_m, tmp7_m; \ 2573 ILVR_H2_SH(in6, in4, in7, in5, s0_m, s1_m); \ 2574 ILVRL_H2_SH(s1_m, s0_m, tmp0_m, tmp1_m); \ 2575 ILVL_H2_SH(in6, in4, in7, in5, s0_m, s1_m); \ 2576 ILVRL_H2_SH(s1_m, s0_m, tmp2_m, tmp3_m); \ 2577 ILVR_H2_SH(in2, in0, in3, in1, s0_m, s1_m); \ 2578 ILVRL_H2_SH(s1_m, s0_m, tmp4_m, tmp5_m); \ 2579 ILVL_H2_SH(in2, in0, in3, in1, s0_m, s1_m); \ 2580 ILVRL_H2_SH(s1_m, s0_m, tmp6_m, tmp7_m); \ 2581 PCKEV_D4(RTYPE, tmp0_m, tmp4_m, tmp1_m, tmp5_m, tmp2_m, tmp6_m, \ 2582 tmp3_m, tmp7_m, out0, out2, out4, out6); \ 2583 out1 = (RTYPE) __msa_pckod_d((v2i64) tmp0_m, (v2i64) tmp4_m); \ 2584 out3 = (RTYPE) __msa_pckod_d((v2i64) tmp1_m, (v2i64) tmp5_m); \ 2585 out5 = (RTYPE) __msa_pckod_d((v2i64) tmp2_m, (v2i64) tmp6_m); \ 2586 out7 = (RTYPE) __msa_pckod_d((v2i64) tmp3_m, (v2i64) tmp7_m); \ 2588 #define TRANSPOSE8x8_UH_UH(...) TRANSPOSE8x8_H(v8u16, __VA_ARGS__) 2589 #define TRANSPOSE8x8_SH_SH(...) TRANSPOSE8x8_H(v8i16, __VA_ARGS__) 2597 #define TRANSPOSE4x4_SW_SW(in0, in1, in2, in3, out0, out1, out2, out3) \ 2599 v4i32 s0_m, s1_m, s2_m, s3_m; \ 2601 ILVRL_W2_SW(in1, in0, s0_m, s1_m); \ 2602 ILVRL_W2_SW(in3, in2, s2_m, s3_m); \ 2604 out0 = (v4i32) __msa_ilvr_d((v2i64) s2_m, (v2i64) s0_m); \ 2605 out1 = (v4i32) __msa_ilvl_d((v2i64) s2_m, (v2i64) s0_m); \ 2606 out2 = (v4i32) __msa_ilvr_d((v2i64) s3_m, (v2i64) s1_m); \ 2607 out3 = (v4i32) __msa_ilvl_d((v2i64) s3_m, (v2i64) s1_m); \ 2624 #define AVE_ST8x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 2626 uint64_t out0_m, out1_m, out2_m, out3_m; \ 2627 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2629 tmp0_m = __msa_ave_u_b((v16u8) in0, (v16u8) in1); \ 2630 tmp1_m = __msa_ave_u_b((v16u8) in2, (v16u8) in3); \ 2631 tmp2_m = __msa_ave_u_b((v16u8) in4, (v16u8) in5); \ 2632 tmp3_m = __msa_ave_u_b((v16u8) in6, (v16u8) in7); \ 2634 out0_m = __msa_copy_u_d((v2i64) tmp0_m, 0); \ 2635 out1_m = __msa_copy_u_d((v2i64) tmp1_m, 0); \ 2636 out2_m = __msa_copy_u_d((v2i64) tmp2_m, 0); \ 2637 out3_m = __msa_copy_u_d((v2i64) tmp3_m, 0); \ 2638 SD4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \ 2655 #define AVE_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 2657 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2659 tmp0_m = __msa_ave_u_b((v16u8) in0, (v16u8) in1); \ 2660 tmp1_m = __msa_ave_u_b((v16u8) in2, (v16u8) in3); \ 2661 tmp2_m = __msa_ave_u_b((v16u8) in4, (v16u8) in5); \ 2662 tmp3_m = __msa_ave_u_b((v16u8) in6, (v16u8) in7); \ 2664 ST_UB4(tmp0_m, tmp1_m, tmp2_m, tmp3_m, pdst, stride); \ 2681 #define AVER_ST8x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 2683 uint64_t out0_m, out1_m, out2_m, out3_m; \ 2684 v16u8 tp0_m, tp1_m, tp2_m, tp3_m; \ 2686 AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2687 tp0_m, tp1_m, tp2_m, tp3_m); \ 2689 out0_m = __msa_copy_u_d((v2i64) tp0_m, 0); \ 2690 out1_m = __msa_copy_u_d((v2i64) tp1_m, 0); \ 2691 out2_m = __msa_copy_u_d((v2i64) tp2_m, 0); \ 2692 out3_m = __msa_copy_u_d((v2i64) tp3_m, 0); \ 2693 SD4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \ 2710 #define AVER_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride) \ 2712 v16u8 t0_m, t1_m, t2_m, t3_m; \ 2714 AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2715 t0_m, t1_m, t2_m, t3_m); \ 2716 ST_UB4(t0_m, t1_m, t2_m, t3_m, pdst, stride); \ 2734 #define AVER_DST_ST8x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2737 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2738 v16u8 dst0_m, dst1_m, dst2_m, dst3_m; \ 2740 LD_UB4(pdst, stride, dst0_m, dst1_m, dst2_m, dst3_m); \ 2741 AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2742 tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ 2743 AVER_ST8x4_UB(dst0_m, tmp0_m, dst1_m, tmp1_m, \ 2744 dst2_m, tmp2_m, dst3_m, tmp3_m, pdst, stride); \ 2762 #define AVER_DST_ST16x4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2765 v16u8 tmp0_m, tmp1_m, tmp2_m, tmp3_m; \ 2766 v16u8 dst0_m, dst1_m, dst2_m, dst3_m; \ 2768 LD_UB4(pdst, stride, dst0_m, dst1_m, dst2_m, dst3_m); \ 2769 AVER_UB4_UB(in0, in1, in2, in3, in4, in5, in6, in7, \ 2770 tmp0_m, tmp1_m, tmp2_m, tmp3_m); \ 2771 AVER_ST16x4_UB(dst0_m, tmp0_m, dst1_m, tmp1_m, \ 2772 dst2_m, tmp2_m, dst3_m, tmp3_m, pdst, stride); \ 2780 #define ADDBLK_ST4x4_UB(in0, in1, in2, in3, pdst, stride) \ 2782 uint32_t src0_m, src1_m, src2_m, src3_m; \ 2783 uint32_t out0_m, out1_m, out2_m, out3_m; \ 2784 v8i16 inp0_m, inp1_m, res0_m, res1_m; \ 2785 v16i8 dst0_m = { 0 }; \ 2786 v16i8 dst1_m = { 0 }; \ 2787 v16i8 zero_m = { 0 }; \ 2789 ILVR_D2_SH(in1, in0, in3, in2, inp0_m, inp1_m) \ 2790 LW4(pdst, stride, src0_m, src1_m, src2_m, src3_m); \ 2791 INSERT_W2_SB(src0_m, src1_m, dst0_m); \ 2792 INSERT_W2_SB(src2_m, src3_m, dst1_m); \ 2793 ILVR_B2_SH(zero_m, dst0_m, zero_m, dst1_m, res0_m, res1_m); \ 2794 ADD2(res0_m, inp0_m, res1_m, inp1_m, res0_m, res1_m); \ 2795 CLIP_SH2_0_255(res0_m, res1_m); \ 2796 PCKEV_B2_SB(res0_m, res0_m, res1_m, res1_m, dst0_m, dst1_m); \ 2798 out0_m = __msa_copy_u_w((v4i32) dst0_m, 0); \ 2799 out1_m = __msa_copy_u_w((v4i32) dst0_m, 1); \ 2800 out2_m = __msa_copy_u_w((v4i32) dst1_m, 0); \ 2801 out3_m = __msa_copy_u_w((v4i32) dst1_m, 1); \ 2802 SW4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \ 2816 #define DPADD_SH3_SH(in0, in1, in2, coeff0, coeff1, coeff2) \ 2821 out0_m = __msa_dotp_s_h((v16i8) in0, (v16i8) coeff0); \ 2822 out0_m = __msa_dpadd_s_h(out0_m, (v16i8) in1, (v16i8) coeff1); \ 2823 tmp1_m = __msa_dotp_s_h((v16i8) in2, (v16i8) coeff2); \ 2824 out0_m = __msa_adds_s_h(out0_m, tmp1_m); \ 2837 #define PCKEV_XORI128_UB(in0, in1) \ 2840 out_m = (v16u8) __msa_pckev_b((v16i8) in1, (v16i8) in0); \ 2841 out_m = (v16u8) __msa_xori_b((v16u8) out_m, 128); \ 2849 #define CONVERT_UB_AVG_ST8x4_UB(in0, in1, in2, in3, \ 2850 dst0, dst1, pdst, stride) \ 2852 v16u8 tmp0_m, tmp1_m; \ 2853 uint8_t *pdst_m = (uint8_t *) (pdst); \ 2855 tmp0_m = PCKEV_XORI128_UB(in0, in1); \ 2856 tmp1_m = PCKEV_XORI128_UB(in2, in3); \ 2857 AVER_UB2_UB(tmp0_m, dst0, tmp1_m, dst1, tmp0_m, tmp1_m); \ 2858 ST8x4_UB(tmp0_m, tmp1_m, pdst_m, stride); \ 2866 #define PCKEV_ST4x4_UB(in0, in1, in2, in3, pdst, stride) \ 2868 uint32_t out0_m, out1_m, out2_m, out3_m; \ 2869 v16i8 tmp0_m, tmp1_m; \ 2871 PCKEV_B2_SB(in1, in0, in3, in2, tmp0_m, tmp1_m); \ 2873 out0_m = __msa_copy_u_w((v4i32) tmp0_m, 0); \ 2874 out1_m = __msa_copy_u_w((v4i32) tmp0_m, 2); \ 2875 out2_m = __msa_copy_u_w((v4i32) tmp1_m, 0); \ 2876 out3_m = __msa_copy_u_w((v4i32) tmp1_m, 2); \ 2878 SW4(out0_m, out1_m, out2_m, out3_m, pdst, stride); \ 2885 #define PCKEV_ST_SB(in0, in1, pdst) \ 2888 tmp_m = __msa_pckev_b((v16i8) in1, (v16i8) in0); \ 2889 ST_SB(tmp_m, (pdst)); \ 2895 #define HORIZ_2TAP_FILT_UH(in0, in1, mask, coeff, shift) \ 2900 tmp0_m = __msa_vshf_b((v16i8) mask, (v16i8) in1, (v16i8) in0); \ 2901 tmp1_m = __msa_dotp_u_h((v16u8) tmp0_m, (v16u8) coeff); \ 2902 tmp1_m = (v8u16) __msa_srari_h((v8i16) tmp1_m, shift); \ 2903 tmp1_m = __msa_sat_u_h(tmp1_m, shift); \