From patchwork Sat Oct 14 08:45:54 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Logan.Lyu" X-Patchwork-Id: 44259 Delivered-To: ffmpegpatchwork2@gmail.com Received: by 2002:a05:6a20:4b15:b0:15d:8365:d4b8 with SMTP id fp21csp1476802pzb; Sat, 14 Oct 2023 01:46:31 -0700 (PDT) X-Google-Smtp-Source: AGHT+IEu/DJllXObPSx0BHpyEcWjIryHcAO2fDX4cijzjjk3oYTM8xyYlwQ211tSi5FVQ4QHU2J4 X-Received: by 2002:a17:907:801:b0:9ba:8ed:ea58 with SMTP id wv1-20020a170907080100b009ba08edea58mr2239186ejb.30.1697273190477; Sat, 14 Oct 2023 01:46:30 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1697273190; cv=none; d=google.com; s=arc-20160816; b=zyEH+2j7YgNCxvkoj+7UKRHvQI2tGGdyqYGK2vwscRMG7fK//mfzi3SVheWxve3F0Y TU+u6yZ9cx8o9eXbcfB3/UdRG/yHZi8NiX8wk5CgVrzQVoYRIioAAlgWQtCzPnVh2/HO er8ltjQ10JkQUihJeuQ4U/SsHhez/zqtuIA0vhxtU5gw6jfzkuixX2QSjuNyNr1N9MAQ r8IS21/EesC2qnoajmQtrdd8aNhLFx9OeZurw7vGEiIAYl5zHQG6zSWxRbDy1TvcXpfJ iR+Dfp806SF/qM/K8Cu52MHV3p7hDQvfwlAqcqyhqw7TAR3qJv/JShUgZesyF5cBrpLG BZ0g== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=sender:errors-to:reply-to:list-subscribe:list-help:list-post :list-archive:list-unsubscribe:list-id:precedence:subject :organization:to:from:user-agent:mime-version:date:message-id :delivered-to; bh=D4uUSYbFwGAIwxNGJEIX4zjOQPzplsZ+Z/CY5UiOG0g=; fh=YOA8vD9MJZuwZ71F/05pj6KdCjf6jQRmzLS+CATXUQk=; b=Ap8cn+NBPVKeqyUpcId+yRDPX0o/55AliAD7XPcrmFwjeO+7sRZ2xEqjJLUiAAeG+K 4QPBctrB8rlQ2+wR2EoS/LL4RrAB8xAQWuUY/f0CPpiGsOGKviwyuvM29RnCXUaanAAn HWDxqeTNo/cD67rp1+gZ7ZwOq60orcQ3How+T9LLrdE1X0iwpt9ZCMBxjUzyAKjLx8Lt vvWR1PM2561stT3biNkn5dmNLJvu8k/lMBF47KwbJ/Ah52pPsye9QCpiqI0higVxqvAS 3l/Uj25VzYu34fxYelgxxtB+pZu+REpAeq2eM1AzmojzHlFl5oBsMPQ0pskCz+xHFLWG YBHQ== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) smtp.mailfrom=ffmpeg-devel-bounces@ffmpeg.org Return-Path: Received: from ffbox0-bg.mplayerhq.hu (ffbox0-bg.ffmpeg.org. [79.124.17.100]) by mx.google.com with ESMTP id cf15-20020a170906b2cf00b009a9f767b62fsi602382ejb.114.2023.10.14.01.46.30; Sat, 14 Oct 2023 01:46:30 -0700 (PDT) Received-SPF: pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) client-ip=79.124.17.100; Authentication-Results: mx.google.com; spf=pass (google.com: domain of ffmpeg-devel-bounces@ffmpeg.org designates 79.124.17.100 as permitted sender) smtp.mailfrom=ffmpeg-devel-bounces@ffmpeg.org Received: from [127.0.1.1] (localhost [127.0.0.1]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 142DF68C9FD; Sat, 14 Oct 2023 11:46:04 +0300 (EEST) X-Original-To: ffmpeg-devel@ffmpeg.org Delivered-To: ffmpeg-devel@ffmpeg.org Received: from smtp-my3-08p62.yunyou.top (smtp-my3-08p62.yunyou.top [60.247.169.62]) by ffbox0-bg.mplayerhq.hu (Postfix) with ESMTP id 2085868C9F2 for ; Sat, 14 Oct 2023 11:45:57 +0300 (EEST) Received: from [192.168.15.106] (unknown [125.121.27.199]) by smtp-my-08.yunyou.top (WestCloudMail) with ESMTPA id DD482AE4A0 for ; Sat, 14 Oct 2023 16:45:54 +0800 (CST) Message-ID: Date: Sat, 14 Oct 2023 16:45:54 +0800 MIME-Version: 1.0 User-Agent: Mozilla Thunderbird From: "Logan.Lyu" To: ffmpeg-devel@ffmpeg.org Organization: myais Subject: [FFmpeg-devel] [PATCH 4/4] lavc/aarch64: new optimization for 8-bit hevc_qpel_hv X-BeenThere: ffmpeg-devel@ffmpeg.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: FFmpeg development discussions and patches List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Reply-To: FFmpeg development discussions and patches Errors-To: ffmpeg-devel-bounces@ffmpeg.org Sender: "ffmpeg-devel" X-TUID: Gw5HG7nsNwjC checkasm bench: put_hevc_qpel_hv4_8_c: 422.1 put_hevc_qpel_hv4_8_i8mm: 101.6 put_hevc_qpel_hv6_8_c: 756.4 put_hevc_qpel_hv6_8_i8mm: 225.9 put_hevc_qpel_hv8_8_c: 1189.9 put_hevc_qpel_hv8_8_i8mm: 296.6 put_hevc_qpel_hv12_8_c: 2407.4 put_hevc_qpel_hv12_8_i8mm: 552.4 put_hevc_qpel_hv16_8_c: 4021.4 put_hevc_qpel_hv16_8_i8mm: 886.6 put_hevc_qpel_hv24_8_c: 8992.1 put_hevc_qpel_hv24_8_i8mm: 1968.9 put_hevc_qpel_hv32_8_c: 15197.9 put_hevc_qpel_hv32_8_i8mm: 3209.4 put_hevc_qpel_hv48_8_c: 32811.1 put_hevc_qpel_hv48_8_i8mm: 7442.1 put_hevc_qpel_hv64_8_c: 58106.1 put_hevc_qpel_hv64_8_i8mm: 12423.9 Co-Authored-By: J. Dekker Signed-off-by: Logan Lyu --- libavcodec/aarch64/hevcdsp_init_aarch64.c | 5 + libavcodec/aarch64/hevcdsp_qpel_neon.S | 397 ++++++++++++++++++++++ 2 files changed, 402 insertions(+) + add sp, sp, x10 // tmp_array without first line + ret +endfunc + +function ff_hevc_put_hevc_qpel_hv48_8_neon_i8mm, export=1 + stp x4, x5, [sp, #-64]! + stp x2, x3, [sp, #16] + stp x0, x1, [sp, #32] + str x30, [sp, #48] + bl X(ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm) + ldp x4, x5, [sp] + ldp x2, x3, [sp, #16] + ldp x0, x1, [sp, #32] + add sp, sp, #48 + add x1, x1, #24 + add x0, x0, #48 + bl X(ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm) + ldr x30, [sp], #16 + ret +endfunc + +function ff_hevc_put_hevc_qpel_hv64_8_neon_i8mm, export=1 + stp x4, x5, [sp, #-64]! + stp x2, x3, [sp, #16] + stp x0, x1, [sp, #32] + str x30, [sp, #48] + mov x6, #32 + bl X(ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm) + ldp x4, x5, [sp] + ldp x2, x3, [sp, #16] + ldp x0, x1, [sp, #32] + add sp, sp, #48 + add x1, x1, #32 + add x0, x0, #64 + mov x6, #32 + bl X(ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm) + ldr x30, [sp], #16 + ret +endfunc + .macro QPEL_UNI_W_HV_HEADER width ldp x14, x15, [sp] // mx, my ldr w13, [sp, #16] // width diff --git a/libavcodec/aarch64/hevcdsp_init_aarch64.c b/libavcodec/aarch64/hevcdsp_init_aarch64.c index f6b4c31d17..7d889efe68 100644 --- a/libavcodec/aarch64/hevcdsp_init_aarch64.c +++ b/libavcodec/aarch64/hevcdsp_init_aarch64.c @@ -208,6 +208,10 @@ NEON8_FNPROTO(qpel_v, (int16_t *dst, const uint8_t *src, ptrdiff_t srcstride, int height, intptr_t mx, intptr_t my, int width),); +NEON8_FNPROTO(qpel_hv, (int16_t *dst, + const uint8_t *src, ptrdiff_t srcstride, + int height, intptr_t mx, intptr_t my, int width), _i8mm); + NEON8_FNPROTO(qpel_uni_v, (uint8_t *dst, ptrdiff_t dststride, const uint8_t *src, ptrdiff_t srcstride, int height, intptr_t mx, intptr_t my, int width),); @@ -335,6 +339,7 @@ av_cold void ff_hevc_dsp_init_aarch64(HEVCDSPContext *c, const int bit_depth) NEON8_FNASSIGN(c->put_hevc_epel_uni, 1, 1, epel_uni_hv, _i8mm); NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 0, 1, epel_uni_w_h ,_i8mm); NEON8_FNASSIGN(c->put_hevc_qpel, 0, 1, qpel_h, _i8mm); + NEON8_FNASSIGN(c->put_hevc_qpel, 1, 1, qpel_hv, _i8mm); NEON8_FNASSIGN(c->put_hevc_qpel_uni, 1, 1, qpel_uni_hv, _i8mm); NEON8_FNASSIGN(c->put_hevc_qpel_uni_w, 0, 1, qpel_uni_w_h, _i8mm); NEON8_FNASSIGN(c->put_hevc_epel_uni_w, 1, 1, epel_uni_w_hv, _i8mm); diff --git a/libavcodec/aarch64/hevcdsp_qpel_neon.S b/libavcodec/aarch64/hevcdsp_qpel_neon.S index eff70d70a4..e4475ba920 100644 --- a/libavcodec/aarch64/hevcdsp_qpel_neon.S +++ b/libavcodec/aarch64/hevcdsp_qpel_neon.S @@ -3070,6 +3070,403 @@ function ff_hevc_put_hevc_qpel_h64_8_neon_i8mm, export=1 ret endfunc + +function ff_hevc_put_hevc_qpel_hv4_8_neon_i8mm, export=1 + add w10, w3, #7 + mov x7, #128 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x5, x30, [sp, #-32]! + stp x0, x3, [sp, #16] + add x0, sp, #32 + sub x1, x1, x2, lsl #1 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h4_8_neon_i8mm) + ldp x5, x30, [sp] + ldp x0, x3, [sp, #16] + add sp, sp, #32 + load_qpel_filterh x5, x4 + ldr d16, [sp] + ldr d17, [sp, x7] + add sp, sp, x7, lsl #1 + ldr d18, [sp] + ldr d19, [sp, x7] + add sp, sp, x7, lsl #1 + ldr d20, [sp] + ldr d21, [sp, x7] + add sp, sp, x7, lsl #1 + ldr d22, [sp] + add sp, sp, x7 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().4h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + subs w3, w3, #1 + st1 {v1.4h}, [x0], x7 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv6_8_neon_i8mm, export=1 + add w10, w3, #7 + mov x7, #128 + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x5, x30, [sp, #-32]! + stp x0, x3, [sp, #16] + add x0, sp, #32 + sub x1, x1, x2, lsl #1 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h6_8_neon_i8mm) + ldp x5, x30, [sp] + mov x8, #120 + ldp x0, x3, [sp, #16] + add sp, sp, #32 + load_qpel_filterh x5, x4 + ldr q16, [sp] + ldr q17, [sp, x7] + add sp, sp, x7, lsl #1 + ldr q18, [sp] + ldr q19, [sp, x7] + add sp, sp, x7, lsl #1 + ldr q20, [sp] + ldr q21, [sp, x7] + add sp, sp, x7, lsl #1 + ldr q22, [sp] + add sp, sp, x7 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + st1 {v1.4h}, [x0], #8 + subs w3, w3, #1 + st1 {v1.s}[2], [x0], x8 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv8_8_neon_i8mm, export=1 + add w10, w3, #7 + lsl x10, x10, #7 + sub x1, x1, x2, lsl #1 + sub sp, sp, x10 // tmp_array + stp x5, x30, [sp, #-32]! + stp x0, x3, [sp, #16] + add x0, sp, #32 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h8_8_neon_i8mm) + ldp x5, x30, [sp] + mov x7, #128 + ldp x0, x3, [sp, #16] + add sp, sp, #32 + load_qpel_filterh x5, x4 + ldr q16, [sp] + ldr q17, [sp, x7] + add sp, sp, x7, lsl #1 + ldr q18, [sp] + ldr q19, [sp, x7] + add sp, sp, x7, lsl #1 + ldr q20, [sp] + ldr q21, [sp, x7] + add sp, sp, x7, lsl #1 + ldr q22, [sp] + add sp, sp, x7 +.macro calc tmp, src0, src1, src2, src3, src4, src5, src6, src7 + ld1 {\tmp\().8h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h}, [x0], x7 +.endm +1: calc_all +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv12_8_neon_i8mm, export=1 + add w10, w3, #7 + lsl x10, x10, #7 + sub x1, x1, x2, lsl #1 + sub sp, sp, x10 // tmp_array + stp x5, x30, [sp, #-32]! + stp x0, x3, [sp, #16] + add x0, sp, #32 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h12_8_neon_i8mm) + ldp x5, x30, [sp] + mov x7, #128 + ldp x0, x3, [sp, #16] + add sp, sp, #32 + load_qpel_filterh x5, x4 + mov x8, #112 + ld1 {v16.8h, v17.8h}, [sp], x7 + ld1 {v18.8h, v19.8h}, [sp], x7 + ld1 {v20.8h, v21.8h}, [sp], x7 + ld1 {v22.8h, v23.8h}, [sp], x7 + ld1 {v24.8h, v25.8h}, [sp], x7 + ld1 {v26.8h, v27.8h}, [sp], x7 + ld1 {v28.8h, v29.8h}, [sp], x7 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + calc_qpelh v2, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn + st1 {v1.8h}, [x0], #16 + subs w3, w3, #1 + st1 {v2.4h}, [x0], x8 +.endm +1: calc_all2 +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv16_8_neon_i8mm, export=1 + add w10, w3, #7 + lsl x10, x10, #7 + sub x1, x1, x2, lsl #1 + sub sp, sp, x10 // tmp_array + stp x5, x30, [sp, #-32]! + stp x0, x3, [sp, #16] + add x3, x3, #7 + add x0, sp, #32 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h16_8_neon_i8mm) + ldp x5, x30, [sp] + mov x7, #128 + ldp x0, x3, [sp, #16] + add sp, sp, #32 + load_qpel_filterh x5, x4 + ld1 {v16.8h, v17.8h}, [sp], x7 + ld1 {v18.8h, v19.8h}, [sp], x7 + ld1 {v20.8h, v21.8h}, [sp], x7 + ld1 {v22.8h, v23.8h}, [sp], x7 + ld1 {v24.8h, v25.8h}, [sp], x7 + ld1 {v26.8h, v27.8h}, [sp], x7 + ld1 {v28.8h, v29.8h}, [sp], x7 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [sp], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + calc_qpelh v2, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn + calc_qpelh2 v2, v3, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h, v2.8h}, [x0], x7 +.endm +1: calc_all2 +.purgem calc +2: ret +endfunc + +function ff_hevc_put_hevc_qpel_hv24_8_neon_i8mm, export=1 + sub sp, sp, #32 + st1 {v8.8b-v11.8b}, [sp] + sub x1, x1, x2, lsl #1 + sub sp, sp, #32 + add w10, w3, #7 + st1 {v12.8b-v15.8b}, [sp] + lsl x10, x10, #7 + sub sp, sp, x10 // tmp_array + stp x5, x30, [sp, #-32]! + stp x0, x3, [sp, #16] + add x0, sp, #32 + add x3, x3, #7 + sub x1, x1, x2 + bl X(ff_hevc_put_hevc_qpel_h24_8_neon_i8mm) + ldp x5, x30, [sp] + mov x7, #128 + ldp x0, x3, [sp, #16] + add sp, sp, #32 + load_qpel_filterh x5, x4 + ld1 {v8.8h-v10.8h}, [sp], x7 + ld1 {v11.8h-v13.8h}, [sp], x7 + ld1 {v14.8h-v16.8h}, [sp], x7 + ld1 {v17.8h-v19.8h}, [sp], x7 + ld1 {v20.8h-v22.8h}, [sp], x7 + ld1 {v23.8h-v25.8h}, [sp], x7 + ld1 {v26.8h-v28.8h}, [sp], x7 +1: ld1 {v29.8h-v31.8h}, [sp], x7 + calc_qpelh v1, v8, v11, v14, v17, v20, v23, v26, v29, sqshrn + calc_qpelh2 v1, v2, v8, v11, v14, v17, v20, v23, v26, v29, sqshrn2 + calc_qpelh v2, v9, v12, v15, v18, v21, v24, v27, v30, sqshrn + calc_qpelh2 v2, v3, v9, v12, v15, v18, v21, v24, v27, v30, sqshrn2 + calc_qpelh v3, v10, v13, v16, v19, v22, v25, v28, v31, sqshrn + calc_qpelh2 v3, v4, v10, v13, v16, v19, v22, v25, v28, v31, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v8.8h-v10.8h}, [sp], x7 + calc_qpelh v1, v11, v14, v17, v20, v23, v26, v29, v8, sqshrn + calc_qpelh2 v1, v2, v11, v14, v17, v20, v23, v26, v29, v8, sqshrn2 + calc_qpelh v2, v12, v15, v18, v21, v24, v27, v30, v9, sqshrn + calc_qpelh2 v2, v3, v12, v15, v18, v21, v24, v27, v30, v9, sqshrn2 + calc_qpelh v3, v13, v16, v19, v22, v25, v28, v31, v10, sqshrn + calc_qpelh2 v3, v4, v13, v16, v19, v22, v25, v28, v31, v10, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v11.8h-v13.8h}, [sp], x7 + calc_qpelh v1, v14, v17, v20, v23, v26, v29, v8, v11, sqshrn + calc_qpelh2 v1, v2, v14, v17, v20, v23, v26, v29, v8, v11, sqshrn2 + calc_qpelh v2, v15, v18, v21, v24, v27, v30, v9, v12, sqshrn + calc_qpelh2 v2, v3, v15, v18, v21, v24, v27, v30, v9, v12, sqshrn2 + calc_qpelh v3, v16, v19, v22, v25, v28, v31, v10, v13, sqshrn + calc_qpelh2 v3, v4, v16, v19, v22, v25, v28, v31, v10, v13, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v14.8h-v16.8h}, [sp], x7 + calc_qpelh v1, v17, v20, v23, v26, v29, v8, v11, v14, sqshrn + calc_qpelh2 v1, v2, v17, v20, v23, v26, v29, v8, v11, v14, sqshrn2 + calc_qpelh v2, v18, v21, v24, v27, v30, v9, v12, v15, sqshrn + calc_qpelh2 v2, v3, v18, v21, v24, v27, v30, v9, v12, v15, sqshrn2 + calc_qpelh v3, v19, v22, v25, v28, v31, v10, v13, v16, sqshrn + calc_qpelh2 v3, v4, v19, v22, v25, v28, v31, v10, v13, v16, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v17.8h-v19.8h}, [sp], x7 + calc_qpelh v1, v20, v23, v26, v29, v8, v11, v14, v17, sqshrn + calc_qpelh2 v1, v2, v20, v23, v26, v29, v8, v11, v14, v17, sqshrn2 + calc_qpelh v2, v21, v24, v27, v30, v9, v12, v15, v18, sqshrn + calc_qpelh2 v2, v3, v21, v24, v27, v30, v9, v12, v15, v18, sqshrn2 + calc_qpelh v3, v22, v25, v28, v31, v10, v13, v16, v19, sqshrn + calc_qpelh2 v3, v4, v22, v25, v28, v31, v10, v13, v16, v19, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v20.8h-v22.8h}, [sp], x7 + calc_qpelh v1, v23, v26, v29, v8, v11, v14, v17, v20, sqshrn + calc_qpelh2 v1, v2, v23, v26, v29, v8, v11, v14, v17, v20, sqshrn2 + calc_qpelh v2, v24, v27, v30, v9, v12, v15, v18, v21, sqshrn + calc_qpelh2 v2, v3, v24, v27, v30, v9, v12, v15, v18, v21, sqshrn2 + calc_qpelh v3, v25, v28, v31, v10, v13, v16, v19, v22, sqshrn + calc_qpelh2 v3, v4, v25, v28, v31, v10, v13, v16, v19, v22, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v23.8h-v25.8h}, [sp], x7 + calc_qpelh v1, v26, v29, v8, v11, v14, v17, v20, v23, sqshrn + calc_qpelh2 v1, v2, v26, v29, v8, v11, v14, v17, v20, v23, sqshrn2 + calc_qpelh v2, v27, v30, v9, v12, v15, v18, v21, v24, sqshrn + calc_qpelh2 v2, v3, v27, v30, v9, v12, v15, v18, v21, v24, sqshrn2 + calc_qpelh v3, v28, v31, v10, v13, v16, v19, v22, v25, sqshrn + calc_qpelh2 v3, v4, v28, v31, v10, v13, v16, v19, v22, v25, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.eq 2f + + ld1 {v26.8h-v28.8h}, [sp], x7 + calc_qpelh v1, v29, v8, v11, v14, v17, v20, v23, v26, sqshrn + calc_qpelh2 v1, v2, v29, v8, v11, v14, v17, v20, v23, v26, sqshrn2 + calc_qpelh v2, v30, v9, v12, v15, v18, v21, v24, v27, sqshrn + calc_qpelh2 v2, v3, v30, v9, v12, v15, v18, v21, v24, v27, sqshrn2 + calc_qpelh v3, v31, v10, v13, v16, v19, v22, v25, v28, sqshrn + calc_qpelh2 v3, v4, v31, v10, v13, v16, v19, v22, v25, v28, sqshrn2 + subs w3, w3, #1 + st1 {v1.8h-v3.8h}, [x0], x7 + b.hi 1b +2: ld1 {v12.8b-v15.8b}, [sp], #32 + ld1 {v8.8b-v11.8b}, [sp], #32 + ret +endfunc + +function ff_hevc_put_hevc_qpel_hv32_8_neon_i8mm, export=1 + add w10, w3, #7 + sub x1, x1, x2, lsl #1 + lsl x10, x10, #7 + sub x1, x1, x2 + sub sp, sp, x10 // tmp_array + stp x5, x30, [sp, #-32]! + stp x0, x3, [sp, #16] + add x3, x3, #7 + add x0, sp, #32 + bl X(ff_hevc_put_hevc_qpel_h32_8_neon_i8mm) + ldp x5, x30, [sp] + mov x7, #128 + ldp x0, x3, [sp, #16] + add sp, sp, #32 + load_qpel_filterh x5, x4 +0: mov x8, sp // src + ld1 {v16.8h, v17.8h}, [x8], x7 + mov w9, w3 // height + ld1 {v18.8h, v19.8h}, [x8], x7 + mov x5, x0 // dst + ld1 {v20.8h, v21.8h}, [x8], x7 + ld1 {v22.8h, v23.8h}, [x8], x7 + ld1 {v24.8h, v25.8h}, [x8], x7 + ld1 {v26.8h, v27.8h}, [x8], x7 + ld1 {v28.8h, v29.8h}, [x8], x7 +.macro calc tmp0, tmp1, src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10, src11, src12, src13, src14, src15 + ld1 {\tmp0\().8h, \tmp1\().8h}, [x8], x7 + calc_qpelh v1, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn + calc_qpelh2 v1, v2, \src0, \src1, \src2, \src3, \src4, \src5, \src6, \src7, sqshrn2 + calc_qpelh v2, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn + calc_qpelh2 v2, v3, \src8, \src9, \src10, \src11, \src12, \src13, \src14, \src15, sqshrn2 + subs x9, x9, #1 + st1 {v1.8h, v2.8h}, [x5], x7 +.endm +1: calc_all2 +.purgem calc +2: add x0, x0, #32 + add sp, sp, #32 + subs w6, w6, #16 + b.hi 0b + add w10, w3, #6 + add sp, sp, #64 // discard rest of first line + lsl x10, x10, #7