| Message ID | NB0-tXQ--3-2@lynne.ee |
|---|---|
| State | New |
| Headers | show |
| Series | [FFmpeg-devel,1/2] x86/tx_float: add support for calling assembly functions from assembly | expand |
| Context | Check | Description |
|---|---|---|
| yinshiyou/configure_loongarch64 | warning | Failed to apply patch |
| andriy/configure_x86 | warning | Failed to apply patch |
On Sat, Sep 03, 2022 at 03:42:36AM +0200, Lynne wrote: > This commit implements an iMDCT in pure assembly. > > This is capable of processing any mod-8 transforms, rather than just > power of two, but since power of two is all we have assembly for > currently, that's what's supported. > It would really benefit if we could somehow use the C code to decide > which function to jump into, but exposing function labels from assebly > into C is anything but easy. > The post-transform loop could probably be improved. > > This was somewhat annoying to write, as we must support arbitrary > strides during runtime. There's a fast branch for stride == 4 bytes > and a slower one which uses vgatherdps. > > Zen 3 benchmarks for stride == 4 for old (av_imdct_half) vs new (av_tx): > > 128pt: > 2811 decicycles in av_tx (imdct),16775916 runs, 1300 skips > 3082 decicycles in av_imdct_half,16776751 runs, 465 skips > > 256pt: > 4920 decicycles in av_tx (imdct),16775820 runs, 1396 skips > 5378 decicycles in av_imdct_half,16776411 runs, 805 skips > > 512pt: > 9668 decicycles in av_tx (imdct),16775774 runs, 1442 skips > 10626 decicycles in av_imdct_half,16775647 runs, 1569 skips > > 1024pt: > 19812 decicycles in av_tx (imdct),16777144 runs, 72 skips > 23036 decicycles in av_imdct_half,16777167 runs, 49 skips > > Patch attached. > x86-32 doesnt digest this very well src/libavutil/x86/tx_float.asm:1540: error: (ASSERT:2) assertion ``8 <= 7'' failed src/libavutil/x86/tx_float.asm:1361: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:721: ... from macro `cglobal' defined here src//libavutil/x86/x86inc.asm:756: ... from macro `cglobal_internal' defined here src//libavutil/x86/x86inc.asm:618: ... from macro `PROLOGUE' defined here src//libavutil/x86/x86inc.asm:304: ... from macro `ASSERT' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7' undefined src/libavutil/x86/tx_float.asm:1361: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:721: ... from macro `cglobal' defined here src//libavutil/x86/x86inc.asm:756: ... from macro `cglobal_internal' defined here src//libavutil/x86/x86inc.asm:620: ... from macro `PROLOGUE' defined here src//libavutil/x86/x86inc.asm:382: ... from macro `ALLOC_STACK' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7' undefined src/libavutil/x86/tx_float.asm:1361: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:721: ... from macro `cglobal' defined here src//libavutil/x86/x86inc.asm:756: ... from macro `cglobal_internal' defined here src//libavutil/x86/x86inc.asm:621: ... from macro `PROLOGUE' defined here src//libavutil/x86/x86inc.asm:273: ... from macro `LOAD_IF_USED' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7' undefined src/libavutil/x86/tx_float.asm:1361: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:721: ... from macro `cglobal' defined here src//libavutil/x86/x86inc.asm:756: ... from macro `cglobal_internal' defined here src//libavutil/x86/x86inc.asm:621: ... from macro `PROLOGUE' defined here src//libavutil/x86/x86inc.asm:273: ... from macro `LOAD_IF_USED' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7' undefined src/libavutil/x86/tx_float.asm:1361: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:721: ... from macro `cglobal' defined here src//libavutil/x86/x86inc.asm:756: ... from macro `cglobal_internal' defined here src//libavutil/x86/x86inc.asm:621: ... from macro `PROLOGUE' defined here src//libavutil/x86/x86inc.asm:273: ... from macro `LOAD_IF_USED' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7' undefined src/libavutil/x86/tx_float.asm:1361: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:721: ... from macro `cglobal' defined here src//libavutil/x86/x86inc.asm:756: ... from macro `cglobal_internal' defined here src//libavutil/x86/x86inc.asm:621: ... from macro `PROLOGUE' defined here src//libavutil/x86/x86inc.asm:273: ... from macro `LOAD_IF_USED' defined here src/libavutil/x86/tx_float.asm:1540: error: invalid combination of opcode and operands src/libavutil/x86/tx_float.asm:1362: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7d' undefined src/libavutil/x86/tx_float.asm:1365: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7d' undefined src/libavutil/x86/tx_float.asm:1366: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r11q' undefined src/libavutil/x86/tx_float.asm:1369: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7d' undefined src/libavutil/x86/tx_float.asm:1380: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1396: ... from macro `movd' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1383: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m15' undefined src/libavutil/x86/tx_float.asm:1384: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1459: ... from macro `pcmpeqd' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m15' undefined src/libavutil/x86/tx_float.asm:1388: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m15' undefined src/libavutil/x86/tx_float.asm:1390: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m8' undefined src/libavutil/x86/tx_float.asm:1394: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m9' undefined src/libavutil/x86/tx_float.asm:1395: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m10' undefined src/libavutil/x86/tx_float.asm:1403: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1414: ... from macro `movshdup' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m11' undefined src/libavutil/x86/tx_float.asm:1404: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1414: ... from macro `movshdup' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m12' undefined src/libavutil/x86/tx_float.asm:1405: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1415: ... from macro `movsldup' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m13' undefined src/libavutil/x86/tx_float.asm:1406: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1415: ... from macro `movsldup' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m10' undefined src/libavutil/x86/tx_float.asm:1408: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1421: ... from macro `mulps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m11' undefined src/libavutil/x86/tx_float.asm:1409: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1421: ... from macro `mulps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m10' undefined src/libavutil/x86/tx_float.asm:1411: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1562: ... from macro `shufps' defined here src//libavutil/x86/x86inc.asm:1262: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m11' undefined src/libavutil/x86/tx_float.asm:1412: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1562: ... from macro `shufps' defined here src//libavutil/x86/x86inc.asm:1262: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m10' undefined src/libavutil/x86/tx_float.asm:1414: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1668: ... from macro `fmaddsubps' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m11' undefined src/libavutil/x86/tx_float.asm:1415: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1668: ... from macro `fmaddsubps' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1417: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1398: ... from macro `movdqa' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src//libavutil/x86/x86inc.asm:1716: ... from macro `vmovdqa' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1418: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1398: ... from macro `movdqa' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src//libavutil/x86/x86inc.asm:1716: ... from macro `vmovdqa' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1422: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1140: ... from macro `add' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1430: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1431: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1432: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1436: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1444: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m8' undefined src/libavutil/x86/tx_float.asm:1449: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1564: ... from macro `shufps' defined here src//libavutil/x86/x86inc.asm:1260: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `m8' undefined src/libavutil/x86/tx_float.asm:1452: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1421: ... from macro `mulps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: invalid combination of opcode and operands src/libavutil/x86/tx_float.asm:1461: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: invalid combination of opcode and operands src/libavutil/x86/tx_float.asm:1462: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r9q' undefined src/libavutil/x86/tx_float.asm:1463: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r10q' undefined src/libavutil/x86/tx_float.asm:1464: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r9q' undefined src/libavutil/x86/tx_float.asm:1468: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1405: ... from macro `movlps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r10q' undefined src/libavutil/x86/tx_float.asm:1469: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1402: ... from macro `movhps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1471: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1472: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r9q' undefined src/libavutil/x86/tx_float.asm:1473: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r10q' undefined src/libavutil/x86/tx_float.asm:1474: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r9q' undefined src/libavutil/x86/tx_float.asm:1478: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1405: ... from macro `movlps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r10q' undefined src/libavutil/x86/tx_float.asm:1479: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1402: ... from macro `movhps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1484: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1152: ... from macro `sub' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1485: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1152: ... from macro `sub' defined here src/libavutil/x86/tx_float.asm:1540: error: invalid combination of opcode and operands src/libavutil/x86/tx_float.asm:1489: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1490: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r11q' undefined src/libavutil/x86/tx_float.asm:1492: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1118: ... from macro `call' defined here src//libavutil/x86/x86inc.asm:1130: ... from macro `call_internal' defined here src/libavutil/x86/tx_float.asm:1540: error: invalid combination of opcode and operands src/libavutil/x86/tx_float.asm:1495: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1499: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1500: ... from macro `IMDCT_FN' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1505: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1506: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1507: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1508: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1530: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1531: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1395: ... from macro `movaps' defined here src//libavutil/x86/x86inc.asm:1264: ... from macro `RUN_AVX_INSTR' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7q' undefined src/libavutil/x86/tx_float.asm:1533: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1140: ... from macro `add' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r8q' undefined src/libavutil/x86/tx_float.asm:1534: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:1152: ... from macro `sub' defined here src/libavutil/x86/tx_float.asm:1540: error: symbol `r7' undefined src/libavutil/x86/tx_float.asm:1537: ... from macro `IMDCT_FN' defined here src//libavutil/x86/x86inc.asm:638: ... from macro `RET' defined here src/ffbuild/common.mak:103: recipe for target 'libavutil/x86/tx_float.o' failed make: *** [libavutil/x86/tx_float.o] Error 1 make: *** Waiting for unfinished jobs.... thx [...]
Sep 3, 2022, 22:55 by michael@niedermayer.cc: > On Sat, Sep 03, 2022 at 03:42:36AM +0200, Lynne wrote: > >> This commit implements an iMDCT in pure assembly. >> >> This is capable of processing any mod-8 transforms, rather than just >> power of two, but since power of two is all we have assembly for >> currently, that's what's supported. >> It would really benefit if we could somehow use the C code to decide >> which function to jump into, but exposing function labels from assebly >> into C is anything but easy. >> The post-transform loop could probably be improved. >> >> This was somewhat annoying to write, as we must support arbitrary >> strides during runtime. There's a fast branch for stride == 4 bytes >> and a slower one which uses vgatherdps. >> >> Zen 3 benchmarks for stride == 4 for old (av_imdct_half) vs new (av_tx): >> >> 128pt: >> 2811 decicycles in av_tx (imdct),16775916 runs, 1300 skips >> 3082 decicycles in av_imdct_half,16776751 runs, 465 skips >> >> 256pt: >> 4920 decicycles in av_tx (imdct),16775820 runs, 1396 skips >> 5378 decicycles in av_imdct_half,16776411 runs, 805 skips >> >> 512pt: >> 9668 decicycles in av_tx (imdct),16775774 runs, 1442 skips >> 10626 decicycles in av_imdct_half,16775647 runs, 1569 skips >> >> 1024pt: >> 19812 decicycles in av_tx (imdct),16777144 runs, 72 skips >> 23036 decicycles in av_imdct_half,16777167 runs, 49 skips >> >> Patch attached. >> > > x86-32 doesnt digest this very well > Thanks for checking, ifdef'd it out of 32bit compiles, also fixed a small issue with asm functions being picked for non-asm calls. Attached.
From fd4a1e81ffcdff2aa03c25abd362ce892e060210 Mon Sep 17 00:00:00 2001 From: Lynne <dev@lynne.ee> Date: Sat, 3 Sep 2022 03:36:40 +0200 Subject: [PATCH 2/2] x86/tx_float: implement inverse MDCT AVX2 assembly This commit implements an iMDCT in pure assembly. This is capable of processing any mod-8 transforms, rather than just power of two, but since power of two is all we have assembly for currently, that's what's supported. It would really benefit if we could somehow use the C code to decide which function to jump into, but exposing function labels from assebly into C is anything but easy. The post-transform loop could probably be improved. This was somewhat annoying to write, as we must support arbitrary strides during runtime. There's a fast branch for stride == 4 bytes and a slower one which uses vgatherdps. Zen 3 benchmarks for stride == 4 for old (av_imdct_half) vs new (av_tx): 128pt: 2811 decicycles in av_tx (imdct),16775916 runs, 1300 skips 3082 decicycles in av_imdct_half,16776751 runs, 465 skips 256pt: 4920 decicycles in av_tx (imdct),16775820 runs, 1396 skips 5378 decicycles in av_imdct_half,16776411 runs, 805 skips 512pt: 9668 decicycles in av_tx (imdct),16775774 runs, 1442 skips 10626 decicycles in av_imdct_half,16775647 runs, 1569 skips 1024pt: 19812 decicycles in av_tx (imdct),16777144 runs, 72 skips 23036 decicycles in av_imdct_half,16777167 runs, 49 skips --- libavutil/tx.c | 19 ++-- libavutil/tx_priv.h | 8 +- libavutil/x86/tx_float.asm | 183 ++++++++++++++++++++++++++++++++++ libavutil/x86/tx_float_init.c | 32 +++++- 4 files changed, 230 insertions(+), 12 deletions(-) diff --git a/libavutil/tx.c b/libavutil/tx.c index 14bfef040f..4b9194faa1 100644 --- a/libavutil/tx.c +++ b/libavutil/tx.c @@ -206,23 +206,24 @@ static void parity_revtab_generator(int *revtab, int n, int inv, int offset, 1, 1, len >> 1, basis, dual_stride, inv_lookup); } -int ff_tx_gen_split_radix_parity_revtab(AVTXContext *s, int invert_lookup, - int basis, int dual_stride) +int ff_tx_gen_split_radix_parity_revtab(AVTXContext *s, int len, int inv, + int inv_lookup, int basis, int dual_stride) { - int len = s->len; - int inv = s->inv; - - if (!(s->map = av_mallocz(len*sizeof(*s->map)))) - return AVERROR(ENOMEM); - basis >>= 1; if (len < basis) return AVERROR(EINVAL); + if (!(s->map = av_mallocz((inv_lookup == -1 ? 2 : 1)*len*sizeof(*s->map)))) + return AVERROR(ENOMEM); + av_assert0(!dual_stride || !(dual_stride & (dual_stride - 1))); av_assert0(dual_stride <= basis); + parity_revtab_generator(s->map, len, inv, 0, 0, 0, len, - basis, dual_stride, invert_lookup); + basis, dual_stride, inv_lookup != 0); + if (inv_lookup == -1) + parity_revtab_generator(s->map + len, len, inv, 0, 0, 0, len, + basis, dual_stride, 0); return 0; } diff --git a/libavutil/tx_priv.h b/libavutil/tx_priv.h index 691942f808..28ac31a597 100644 --- a/libavutil/tx_priv.h +++ b/libavutil/tx_priv.h @@ -288,9 +288,13 @@ int ff_tx_gen_ptwo_inplace_revtab_idx(AVTXContext *s); * functions in AVX mode. * * If length is smaller than basis/2 this function will not do anything. + * + * If inv_lookup is set to 1, it will flip the lookup from out[map[i]] = src[i] + * to out[i] = src[map[i]]. If set to -1, will generate 2 maps, the first one + * flipped, the second one regular. */ -int ff_tx_gen_split_radix_parity_revtab(AVTXContext *s, int invert_lookup, - int basis, int dual_stride); +int ff_tx_gen_split_radix_parity_revtab(AVTXContext *s, int len, int inv, + int inv_lookup, int basis, int dual_stride); /* Typed init function to initialize shared tables. Will initialize all tables * for all factors of a length. */ diff --git a/libavutil/x86/tx_float.asm b/libavutil/x86/tx_float.asm index 791ad7b322..7ce671c437 100644 --- a/libavutil/x86/tx_float.asm +++ b/libavutil/x86/tx_float.asm @@ -1355,3 +1355,186 @@ FFT_SPLIT_RADIX_FN avx2, 0 FFT_SPLIT_RADIX_FN avx2, 1 %endif %endif + +%macro IMDCT_FN 1 +INIT_YMM %1 +cglobal mdct_sr_inv_float, 4, 12, 16, 288, ctx, out, in, stride, len, lut, exp, t1, t2, t3, t4, t5 + movsxd lenq, dword [ctxq + AVTXContext.len] + mov expq, [ctxq + AVTXContext.exp] + + lea t1d, [lend - 1] + imul t1d, strided + + PUSH outq ; backup original output + mov t5q, [ctxq + AVTXContext.fn] ; subtransform's jump point + PUSH ctxq ; backup original context + mov ctxq, [ctxq + AVTXContext.sub] ; load subtransform's context + mov lutq, [ctxq + AVTXContext.map] ; load subtransform's map + + cmp strideq, 4 + je .stride4 + + shl strideq, 1 + movd xm4, strided + vpbroadcastd m4, xm4 ; stride splatted + movd xm5, t1d + vpbroadcastd m5, xm5 ; offset splatted + + mov t2q, outq ; don't modify the original output + pcmpeqd m15, m15 ; set all bits to 1 + +.stridex_pre: + pmulld m2, m4, [lutq] ; multiply by stride + movaps m0, m15 + psubd m3, m5, m2 ; subtract from offset + movaps m1, m15 + vgatherdps m6, [inq + m2], m0 ; im + vgatherdps m7, [inq + m3], m1 ; re + + movaps m8, [expq + 0*mmsize] ; tab 1 + movaps m9, [expq + 1*mmsize] ; tab 2 + + unpcklps m0, m7, m6 ; re, im, re, im + unpckhps m1, m7, m6 ; re, im, re, im + + vperm2f128 m2, m1, m0, 0x02 ; output order + vperm2f128 m3, m1, m0, 0x13 ; output order + + movshdup m10, m8 ; tab 1 imim + movshdup m11, m9 ; tab 2 imim + movsldup m12, m8 ; tab 1 rere + movsldup m13, m9 ; tab 2 rere + + mulps m10, m2 ; 1 reim * imim + mulps m11, m3 ; 2 reim * imim + + shufps m10, m10, q2301 + shufps m11, m11, q2301 + + fmaddsubps m10, m12, m2, m10 + fmaddsubps m11, m13, m3, m11 + + mova [t2q + 0*mmsize], m10 + mova [t2q + 1*mmsize], m11 + + add expq, mmsize*2 + add lutq, mmsize + add t2q, mmsize*2 + sub lenq, mmsize/2 + jg .stridex_pre + jmp .transform + +.stride4: + lea expq, [expq + lenq*4] + lea lutq, [lutq + lenq*2] + lea t1q, [inq + t1q] + lea t1q, [t1q + strideq - mmsize] + lea t2q, [lenq*2 - mmsize/2] + +.stride4_pre: + movaps m4, [inq] + movaps m3, [t1q] + + movsldup m1, m4 ; im im, im im + movshdup m0, m3 ; re re, re re + movshdup m4, m4 ; re re, re re (2) + movsldup m3, m3 ; im im, im im (2) + + movaps m2, [expq] ; tab + movaps m5, [expq + 2*t2q] ; tab (2) + + vpermpd m0, m0, q0123 ; flip + shufps m7, m2, m2, q2301 + vpermpd m4, m4, q0123 ; flip (2) + shufps m8, m5, m5, q2301 + + mulps m1, m7 ; im im * tab.reim + mulps m3, m8 ; im im * tab.reim (2) + + fmaddsubps m0, m0, m2, m1 + fmaddsubps m4, m4, m5, m3 + + vextractf128 xm3, m0, 1 + vextractf128 xm6, m4, 1 + + ; scatter + movsxd strideq, dword [lutq + 0*4] + movsxd lenq, dword [lutq + 1*4] + movsxd t3q, dword [lutq + 2*4] + movsxd t4q, dword [lutq + 3*4] + + movlps [outq + strideq*8], xm0 + movhps [outq + lenq*8], xm0 + movlps [outq + t3q*8], xm3 + movhps [outq + t4q*8], xm3 + + movsxd strideq, dword [lutq + 0*4 + t2q] + movsxd lenq, dword [lutq + 1*4 + t2q] + movsxd t3q, dword [lutq + 2*4 + t2q] + movsxd t4q, dword [lutq + 3*4 + t2q] + + movlps [outq + strideq*8], xm4 + movhps [outq + lenq*8], xm4 + movlps [outq + t3q*8], xm6 + movhps [outq + t4q*8], xm6 + + add lutq, mmsize/2 + add expq, mmsize + add inq, mmsize + sub t1q, mmsize + sub t2q, mmsize + jg .stride4_pre + +.transform: + movsxd lenq, dword [ctxq + AVTXContext.len] + mov t2q, lenq ; target length (for ptwo transforms) + mov inq, outq ; in-place transform + call t5q ; call the FFT + + POP ctxq ; restore original context + movsxd lenq, dword [ctxq + AVTXContext.len] + mov expq, [ctxq + AVTXContext.exp] + lea expq, [expq + lenq*4] + + lea t1q, [lenq*2] ; high + lea t2q, [lenq*2 - mmsize] ; low + + POP outq + +.post: + movaps m2, [expq + t1q] ; tab h + movaps m3, [expq + t2q] ; tab l + movaps m0, [outq + t1q] ; in h + movaps m1, [outq + t2q] ; in l + + movshdup m4, m2 ; tab h imim + movshdup m5, m3 ; tab l imim + movsldup m6, m2 ; tab h rere + movsldup m7, m3 ; tab l rere + + shufps m2, m0, m0, q2301 ; in h imre + shufps m3, m1, m1, q2301 ; in l imre + + mulps m6, m0 + mulps m7, m1 + + fmaddsubps m4, m4, m2, m6 + fmaddsubps m5, m5, m3, m7 + + vpermpd m3, m5, q0123 ; flip + vpermpd m2, m4, q0123 ; flip + + blendps m1, m2, m5, 01010101b + blendps m0, m3, m4, 01010101b + + movaps [outq + t2q], m1 + movaps [outq + t1q], m0 + + add t1q, mmsize + sub t2q, mmsize + jge .post + + RET +%endmacro + +IMDCT_FN avx2 diff --git a/libavutil/x86/tx_float_init.c b/libavutil/x86/tx_float_init.c index 9fd98103c8..7a462bb06a 100644 --- a/libavutil/x86/tx_float_init.c +++ b/libavutil/x86/tx_float_init.c @@ -43,6 +43,8 @@ TX_DECL_FN(fft_sr_ns, fma3) TX_DECL_FN(fft_sr, avx2) TX_DECL_FN(fft_sr_ns, avx2) +TX_DECL_FN(mdct_sr_inv, avx2) + TX_DECL_FN(fft8_asm, sse3) TX_DECL_FN(fft8_asm, avx) TX_DECL_FN(fft16_asm, avx) @@ -65,13 +67,38 @@ static av_cold int b ##basis## _i ##interleave(AVTXContext *s, \ if (cd->max_len == 2) \ return ff_tx_gen_ptwo_revtab(s, inv_lookup); \ else \ - return ff_tx_gen_split_radix_parity_revtab(s, inv_lookup, \ + return ff_tx_gen_split_radix_parity_revtab(s, len, inv, inv_lookup, \ basis, interleave); \ } DECL_INIT_FN(8, 0) DECL_INIT_FN(8, 2) +static av_cold int m_inv_init(AVTXContext *s, const FFTXCodelet *cd, + uint64_t flags, FFTXCodeletOptions *opts, + int len, int inv, const void *scale) +{ + int ret; + FFTXCodeletOptions sub_opts = { .invert_lookup = -1 }; + + s->scale_d = *((SCALE_TYPE *)scale); + s->scale_f = s->scale_d; + + flags &= ~FF_TX_OUT_OF_PLACE; /* We want the subtransform to be */ + flags |= AV_TX_INPLACE; /* in-place */ + flags |= FF_TX_PRESHUFFLE; /* This function handles the permute step */ + flags |= FF_TX_ASM_CALL; /* We want an assembly function, not C */ + + if ((ret = ff_tx_init_subtx(s, TX_TYPE(FFT), flags, &sub_opts, len >> 1, + inv, scale))) + return ret; + + if ((ret = ff_tx_mdct_gen_exp_float(s, s->sub->map))) + return ret; + + return 0; +} + const FFTXCodelet * const ff_tx_codelet_list_float_x86[] = { TX_DEF(fft2, FFT, 2, 2, 2, 0, 128, NULL, sse3, SSE3, AV_TX_INPLACE, 0), TX_DEF(fft2, FFT, 2, 2, 2, 0, 192, b8_i0, sse3, SSE3, AV_TX_INPLACE | FF_TX_PRESHUFFLE, 0), @@ -98,6 +125,9 @@ const FFTXCodelet * const ff_tx_codelet_list_float_x86[] = { TX_DEF(fft16_ns, FFT, 16, 16, 2, 0, 352, b8_i2, fma3, FMA3, AV_TX_INPLACE | FF_TX_PRESHUFFLE, AV_CPU_FLAG_AVXSLOW), + TX_DEF(mdct_sr_inv, MDCT, 16, TX_LEN_UNLIMITED, 2, TX_FACTOR_ANY, 384, m_inv_init, avx2, AVX2, + FF_TX_INVERSE_ONLY, AV_CPU_FLAG_AVXSLOW | AV_CPU_FLAG_SLOW_GATHER), + #if ARCH_X86_64 TX_DEF(fft32, FFT, 32, 32, 2, 0, 256, b8_i2, avx, AVX, AV_TX_INPLACE, AV_CPU_FLAG_AVXSLOW), TX_DEF(fft32_asm, FFT, 32, 32, 2, 0, 320, b8_i2, avx, AVX, -- 2.37.2.609.g9ff673ca1a