diff mbox series

[FFmpeg-devel,06/21] aarch64: hevc: Implement a neon version of put_hevc_epel_h*_8

Message ID 20240325150243.59058-7-martin@martin.st
State Accepted
Commit 6d384298ece19af357b7fd3f683afff0d3f40c56
Headers show
Series aarch64: hevc: Add missing hevc_pel NEON functions | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Martin Storsjö March 25, 2024, 3:02 p.m. UTC 
AWS Graviton 3:
put_hevc_epel_h4_8_c: 64.7
put_hevc_epel_h4_8_neon: 25.0
put_hevc_epel_h4_8_i8mm: 21.2
put_hevc_epel_h6_8_c: 130.0
put_hevc_epel_h6_8_neon: 40.7
put_hevc_epel_h6_8_i8mm: 36.5
put_hevc_epel_h8_8_c: 209.0
put_hevc_epel_h8_8_neon: 45.2
put_hevc_epel_h8_8_i8mm: 41.2
put_hevc_epel_h12_8_c: 465.5
put_hevc_epel_h12_8_neon: 104.5
put_hevc_epel_h12_8_i8mm: 86.5
put_hevc_epel_h16_8_c: 830.7
put_hevc_epel_h16_8_neon: 134.2
put_hevc_epel_h16_8_i8mm: 114.0
put_hevc_epel_h24_8_c: 1844.7
put_hevc_epel_h24_8_neon: 282.2
put_hevc_epel_h24_8_i8mm: 277.2
put_hevc_epel_h32_8_c: 3227.5
put_hevc_epel_h32_8_neon: 501.5
put_hevc_epel_h32_8_i8mm: 396.0
put_hevc_epel_h48_8_c: 7229.2
put_hevc_epel_h48_8_neon: 1120.2
put_hevc_epel_h48_8_i8mm: 901.2
put_hevc_epel_h64_8_c: 12869.0
put_hevc_epel_h64_8_neon: 1999.2
put_hevc_epel_h64_8_i8mm: 1610.5
---
 libavcodec/aarch64/hevcdsp_epel_neon.S    | 194 +++++++++++++++++++++-
 libavcodec/aarch64/hevcdsp_init_aarch64.c |  17 ++
 2 files changed, 209 insertions(+), 2 deletions(-)
diff mbox series

Patch

diff --git a/libavcodec/aarch64/hevcdsp_epel_neon.S b/libavcodec/aarch64/hevcdsp_epel_neon.S
index d3f0a26f79..419e83529a 100644
--- a/libavcodec/aarch64/hevcdsp_epel_neon.S
+++ b/libavcodec/aarch64/hevcdsp_epel_neon.S
@@ -1321,8 +1321,6 @@  function ff_hevc_put_hevc_epel_uni_v64_8_neon, export=1
         ret
 endfunc
 
-#if HAVE_I8MM
-ENABLE_I8MM
 
 .macro EPEL_H_HEADER
         movrel          x5, epel_filters
@@ -1332,6 +1330,198 @@  ENABLE_I8MM
         mov             x10, #(MAX_PB_SIZE * 2)
 .endm
 
+function ff_hevc_put_hevc_epel_h4_8_neon, export=1
+        EPEL_H_HEADER
+        sxtl            v0.8h,   v30.8b
+1:      ld1             {v4.8b}, [x1], x2
+        subs            w3,  w3,  #1   // height
+        uxtl            v4.8h,   v4.8b
+        ext             v5.16b,  v4.16b,  v4.16b,  #2
+        ext             v6.16b,  v4.16b,  v4.16b,  #4
+        ext             v7.16b,  v4.16b,  v4.16b,  #6
+        mul             v16.4h,  v4.4h,   v0.h[0]
+        mla             v16.4h,  v5.4h,   v0.h[1]
+        mla             v16.4h,  v6.4h,   v0.h[2]
+        mla             v16.4h,  v7.4h,   v0.h[3]
+        st1             {v16.4h}, [x0], x10
+        b.ne            1b
+        ret
+endfunc
+
+function ff_hevc_put_hevc_epel_h6_8_neon, export=1
+        EPEL_H_HEADER
+        sxtl            v0.8h,   v30.8b
+        add             x6,  x0,  #8
+1:      ld1             {v3.16b},  [x1], x2
+        subs            w3,  w3,  #1   // height
+        uxtl2           v4.8h,   v3.16b
+        uxtl            v3.8h,   v3.8b
+        ext             v5.16b,  v3.16b,  v4.16b,  #2
+        ext             v6.16b,  v3.16b,  v4.16b,  #4
+        ext             v7.16b,  v3.16b,  v4.16b,  #6
+        mul             v16.8h,  v3.8h,   v0.h[0]
+        mla             v16.8h,  v5.8h,   v0.h[1]
+        mla             v16.8h,  v6.8h,   v0.h[2]
+        mla             v16.8h,  v7.8h,   v0.h[3]
+        st1             {v16.4h},   [x0], x10
+        st1             {v16.s}[2], [x6], x10
+        b.ne            1b
+        ret
+endfunc
+
+function ff_hevc_put_hevc_epel_h8_8_neon, export=1
+        EPEL_H_HEADER
+        sxtl            v0.8h,   v30.8b
+1:      ld1             {v3.16b},  [x1], x2
+        subs            w3,  w3,  #1   // height
+        uxtl2           v4.8h,   v3.16b
+        uxtl            v3.8h,   v3.8b
+        ext             v5.16b,  v3.16b,  v4.16b,  #2
+        ext             v6.16b,  v3.16b,  v4.16b,  #4
+        ext             v7.16b,  v3.16b,  v4.16b,  #6
+        mul             v16.8h,  v3.8h,   v0.h[0]
+        mla             v16.8h,  v5.8h,   v0.h[1]
+        mla             v16.8h,  v6.8h,   v0.h[2]
+        mla             v16.8h,  v7.8h,   v0.h[3]
+        st1             {v16.8h},   [x0], x10
+        b.ne            1b
+        ret
+endfunc
+
+function ff_hevc_put_hevc_epel_h12_8_neon, export=1
+        EPEL_H_HEADER
+        add             x6,  x0,  #16
+        sxtl            v0.8h,   v30.8b
+1:      ld1             {v3.16b}, [x1], x2
+        subs            w3,  w3,  #1   // height
+        uxtl2           v4.8h,   v3.16b
+        uxtl            v3.8h,   v3.8b
+        ext             v5.16b,  v3.16b,  v4.16b,  #2
+        ext             v6.16b,  v3.16b,  v4.16b,  #4
+        ext             v7.16b,  v3.16b,  v4.16b,  #6
+        ext             v20.16b, v4.16b,  v4.16b,  #2
+        ext             v21.16b, v4.16b,  v4.16b,  #4
+        ext             v22.16b, v4.16b,  v4.16b,  #6
+        mul             v16.8h,  v3.8h,   v0.h[0]
+        mla             v16.8h,  v5.8h,   v0.h[1]
+        mla             v16.8h,  v6.8h,   v0.h[2]
+        mla             v16.8h,  v7.8h,   v0.h[3]
+        mul             v17.4h,  v4.4h,   v0.h[0]
+        mla             v17.4h,  v20.4h,  v0.h[1]
+        mla             v17.4h,  v21.4h,  v0.h[2]
+        mla             v17.4h,  v22.4h,  v0.h[3]
+        st1             {v16.8h}, [x0], x10
+        st1             {v17.4h}, [x6], x10
+        b.ne            1b
+        ret
+endfunc
+
+function ff_hevc_put_hevc_epel_h16_8_neon, export=1
+        EPEL_H_HEADER
+        sxtl            v0.8h,   v30.8b
+1:      ld1             {v1.8b, v2.8b, v3.8b}, [x1], x2
+        subs            w3,  w3,  #1   // height
+        uxtl            v1.8h,   v1.8b
+        uxtl            v2.8h,   v2.8b
+        uxtl            v3.8h,   v3.8b
+        ext             v5.16b,  v1.16b,  v2.16b,  #2
+        ext             v6.16b,  v1.16b,  v2.16b,  #4
+        ext             v7.16b,  v1.16b,  v2.16b,  #6
+        ext             v20.16b, v2.16b,  v3.16b,  #2
+        ext             v21.16b, v2.16b,  v3.16b,  #4
+        ext             v22.16b, v2.16b,  v3.16b,  #6
+        mul             v16.8h,  v1.8h,   v0.h[0]
+        mla             v16.8h,  v5.8h,   v0.h[1]
+        mla             v16.8h,  v6.8h,   v0.h[2]
+        mla             v16.8h,  v7.8h,   v0.h[3]
+        mul             v17.8h,  v2.8h,   v0.h[0]
+        mla             v17.8h,  v20.8h,  v0.h[1]
+        mla             v17.8h,  v21.8h,  v0.h[2]
+        mla             v17.8h,  v22.8h,  v0.h[3]
+        st1             {v16.8h, v17.8h}, [x0], x10
+        b.ne            1b
+        ret
+endfunc
+
+function ff_hevc_put_hevc_epel_h24_8_neon, export=1
+        EPEL_H_HEADER
+        sxtl            v0.8h,   v30.8b
+1:      ld1             {v1.8b, v2.8b, v3.8b, v4.8b}, [x1], x2
+        subs            w3,  w3,  #1   // height
+        uxtl            v1.8h,   v1.8b
+        uxtl            v2.8h,   v2.8b
+        uxtl            v3.8h,   v3.8b
+        uxtl            v4.8h,   v4.8b
+        ext             v5.16b,  v1.16b,  v2.16b,  #2
+        ext             v6.16b,  v1.16b,  v2.16b,  #4
+        ext             v7.16b,  v1.16b,  v2.16b,  #6
+        ext             v20.16b, v2.16b,  v3.16b,  #2
+        ext             v21.16b, v2.16b,  v3.16b,  #4
+        ext             v22.16b, v2.16b,  v3.16b,  #6
+        ext             v23.16b, v3.16b,  v4.16b,  #2
+        ext             v24.16b, v3.16b,  v4.16b,  #4
+        ext             v25.16b, v3.16b,  v4.16b,  #6
+        mul             v16.8h,  v1.8h,   v0.h[0]
+        mla             v16.8h,  v5.8h,   v0.h[1]
+        mla             v16.8h,  v6.8h,   v0.h[2]
+        mla             v16.8h,  v7.8h,   v0.h[3]
+        mul             v17.8h,  v2.8h,   v0.h[0]
+        mla             v17.8h,  v20.8h,  v0.h[1]
+        mla             v17.8h,  v21.8h,  v0.h[2]
+        mla             v17.8h,  v22.8h,  v0.h[3]
+        mul             v18.8h,  v3.8h,   v0.h[0]
+        mla             v18.8h,  v23.8h,  v0.h[1]
+        mla             v18.8h,  v24.8h,  v0.h[2]
+        mla             v18.8h,  v25.8h,  v0.h[3]
+        st1             {v16.8h, v17.8h, v18.8h}, [x0], x10
+        b.ne            1b
+        ret
+endfunc
+
+function ff_hevc_put_hevc_epel_h32_8_neon, export=1
+        EPEL_H_HEADER
+        ld1             {v1.8b}, [x1], #8
+        sub             x2,  x2,  w6, uxtw    // decrement src stride
+        mov             w7,  w6               // original width
+        sub             x2,  x2,  #8          // decrement src stride
+        sub             x10, x10, w6, uxtw #1 // decrement dst stride
+        sxtl            v0.8h,   v30.8b
+        uxtl            v1.8h,   v1.8b
+1:      ld1             {v2.8b, v3.8b}, [x1], #16
+        subs            w6,  w6,  #16   // width
+        uxtl            v2.8h,   v2.8b
+        uxtl            v3.8h,   v3.8b
+        ext             v5.16b,  v1.16b,  v2.16b,  #2
+        ext             v6.16b,  v1.16b,  v2.16b,  #4
+        ext             v7.16b,  v1.16b,  v2.16b,  #6
+        ext             v20.16b, v2.16b,  v3.16b,  #2
+        ext             v21.16b, v2.16b,  v3.16b,  #4
+        ext             v22.16b, v2.16b,  v3.16b,  #6
+        mul             v16.8h,  v1.8h,   v0.h[0]
+        mla             v16.8h,  v5.8h,   v0.h[1]
+        mla             v16.8h,  v6.8h,   v0.h[2]
+        mla             v16.8h,  v7.8h,   v0.h[3]
+        mul             v17.8h,  v2.8h,   v0.h[0]
+        mla             v17.8h,  v20.8h,  v0.h[1]
+        mla             v17.8h,  v21.8h,  v0.h[2]
+        mla             v17.8h,  v22.8h,  v0.h[3]
+        st1             {v16.8h, v17.8h}, [x0], #32
+        mov             v1.16b,  v3.16b
+        b.gt            1b
+        subs            w3,  w3,  #1   // height
+        add             x1,  x1,  x2
+        b.le            9f
+        ld1             {v1.8b}, [x1], #8
+        mov             w6,  w7
+        add             x0,  x0,  x10
+        uxtl            v1.8h,   v1.8b
+        b               1b
+9:
+        ret
+endfunc
+
+#if HAVE_I8MM
+ENABLE_I8MM
 function ff_hevc_put_hevc_epel_h4_8_neon_i8mm, export=1
         EPEL_H_HEADER
 1:      ld1             {v4.8b}, [x1], x2
diff --git a/libavcodec/aarch64/hevcdsp_init_aarch64.c b/libavcodec/aarch64/hevcdsp_init_aarch64.c
index 1e9f5e32db..ece911b8d4 100644
--- a/libavcodec/aarch64/hevcdsp_init_aarch64.c
+++ b/libavcodec/aarch64/hevcdsp_init_aarch64.c
@@ -223,6 +223,10 @@  NEON8_FNPROTO_PARTIAL_4(qpel_uni_w_v, (uint8_t *_dst,  ptrdiff_t _dststride,
         int height, int denom, int wx, int ox,
         intptr_t mx, intptr_t my, int width),);
 
+NEON8_FNPROTO(epel_h, (int16_t *dst,
+        const uint8_t *_src, ptrdiff_t _srcstride,
+        int height, intptr_t mx, intptr_t my, int width),);
+
 NEON8_FNPROTO(epel_h, (int16_t *dst,
         const uint8_t *_src, ptrdiff_t _srcstride,
         int height, intptr_t mx, intptr_t my, int width), _i8mm);
@@ -290,6 +294,17 @@  NEON8_FNPROTO(qpel_bi_hv, (uint8_t *dst, ptrdiff_t dststride,
         member[8][v][h] = ff_hevc_put_hevc_##fn##48_8_neon##ext; \
         member[9][v][h] = ff_hevc_put_hevc_##fn##64_8_neon##ext;
 
+#define NEON8_FNASSIGN_SHARED_32(member, v, h, fn, ext) \
+        member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext;  \
+        member[2][v][h] = ff_hevc_put_hevc_##fn##6_8_neon##ext;  \
+        member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext;  \
+        member[4][v][h] = ff_hevc_put_hevc_##fn##12_8_neon##ext; \
+        member[5][v][h] = ff_hevc_put_hevc_##fn##16_8_neon##ext; \
+        member[6][v][h] = ff_hevc_put_hevc_##fn##24_8_neon##ext; \
+        member[7][v][h] =                                        \
+        member[8][v][h] =                                        \
+        member[9][v][h] = ff_hevc_put_hevc_##fn##32_8_neon##ext;
+
 #define NEON8_FNASSIGN_PARTIAL_4(member, v, h, fn, ext) \
         member[1][v][h] = ff_hevc_put_hevc_##fn##4_8_neon##ext;  \
         member[3][v][h] = ff_hevc_put_hevc_##fn##8_8_neon##ext;  \
@@ -384,6 +399,8 @@  av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth)
         NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 0, epel_uni_w_v,);
         NEON8_FNASSIGN_PARTIAL_4(c->put_hevc_qpel_uni_w, 1, 0, qpel_uni_w_v,);
 
+        NEON8_FNASSIGN_SHARED_32(c->put_hevc_epel, 0, 1, epel_h,);
+
         if (have_i8mm(cpu_flags)) {
             NEON8_FNASSIGN(c->put_hevc_epel, 0, 1, epel_h, _i8mm);
             NEON8_FNASSIGN(c->put_hevc_epel, 1, 1, epel_hv, _i8mm);