@@ -6,5 +6,6 @@ OBJS-$(CONFIG_VVC_DECODER) += x86/vvc/vvcdsp_init.o \
X86ASM-OBJS-$(CONFIG_VVC_DECODER) += x86/vvc/vvc_alf.o \
x86/vvc/vvc_dmvr.o \
x86/vvc/vvc_mc.o \
+ x86/vvc/vvc_of.o \
x86/vvc/vvc_sad.o \
x86/h26x/h2656_inter.o
new file mode 100644
@@ -0,0 +1,385 @@
+; /*
+; * Provide AVX2 luma optical flow functions for VVC decoding
+; * Copyright (c) 2024 Nuo Mi
+; *
+; * This file is part of FFmpeg.
+; *
+; * FFmpeg is free software; you can redistribute it and/or
+; * modify it under the terms of the GNU Lesser General Public
+; * License as published by the Free Software Foundation; either
+; * version 2.1 of the License, or (at your option) any later version.
+; *
+; * FFmpeg is distributed in the hope that it will be useful,
+; * but WITHOUT ANY WARRANTY; without even the implied warranty of
+; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+; * Lesser General Public License for more details.
+; *
+; * You should have received a copy of the GNU Lesser General Public
+; * License along with FFmpeg; if not, write to the Free Software
+; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+; */
+%include "libavutil/x86/x86util.asm"
+
+%define MAX_PB_SIZE 128
+%define SRC_STRIDE (MAX_PB_SIZE * 2)
+%define SRC_PS 2 ; source pixel size, sizeof(int16_t)
+%define BDOF_STACK_SIZE 10 ; (4 + 1) * 2, 4 lines + the first line, *2 for h and v
+%define bdof_stack_offset(line) ((line) * 2 % BDOF_STACK_SIZE * mmsize)
+%define SHIFT 6
+%define SHIFT2 4
+
+SECTION_RODATA 32
+pd_15 times 8 dd 15
+pd_m15 times 8 dd -15
+
+pb_shuffle_w8 times 2 db 0, 1, 0xff, 0xff, 8, 9, 0xff, 0xff, 6, 7, 0xff, 0xff, 14, 15, 0xff, 0xff
+pb_shuffle_w16 times 2 db 0, 1, 0xff, 0xff, 6, 7, 0xff, 0xff, 8, 9, 0xff, 0xff, 14, 15, 0xff, 0xff
+pd_perm_w16 dd 0, 2, 1, 4, 3, 6, 5, 7
+%if ARCH_X86_64
+
+%if HAVE_AVX2_EXTERNAL
+
+SECTION .text
+
+INIT_YMM avx2
+
+; dst = (src0 >> shift) - (src1 >> shift)
+%macro DIFF 5 ; dst, src0, src1, shift, tmp
+ psraw %1, %2, %4
+ psraw %5, %3, %4
+ psubw %1, %5
+%endmacro
+
+%macro LOAD_GRAD_H 4 ; dst, src, off, tmp
+ movu %1, [%2 + %3 + 2 * SRC_PS]
+ movu %4, [%2 + %3]
+
+ DIFF %1, %1, %4, SHIFT, %4
+%endmacro
+
+%macro SUM_GRAD 2 ;(dst/grad0, grad1)
+ paddw %1, %2
+ psraw %1, 1 ; shift3
+%endmacro
+
+%macro APPLY_BDOF_MIN_BLOCK_LINE 5 ; dst, vx, vy, tmp, line_num
+%define off bdof_stack_offset(%5)
+ pmullw %1, %2, [rsp + off + 0 * mmsize] ; vx * (gradient_h[0] - gradient_h[1])
+ pmullw %4, %3, [rsp + off + 1 * mmsize] ; vy * (gradient_v[0] - gradient_v[1])
+ paddw %1, [src0q + (%5 + 1) * SRC_STRIDE + SRC_PS]
+ paddw %4, [src1q + (%5 + 1) * SRC_STRIDE + SRC_PS]
+ paddsw %1, %4 ; src0[x] + src1[x] + bdof_offset
+ pmulhrsw %1, m11
+ CLIPW %1, m9, m10
+%endmacro
+
+%macro SAVE_8BPC 2 ; dst, src
+ packuswb m%2, m%2
+ vpermq m%2, m%2, q0020
+
+ cmp wd, 16
+ je %%w16
+ movq %1, xm%2
+ jmp %%wend
+%%w16:
+ movu %1, xm%2
+%%wend:
+%endmacro
+
+%macro SAVE_16BPC 2 ; dst, src
+ cmp wd, 16
+ je %%w16
+ movu %1, xm%2
+ jmp %%wend
+%%w16:
+ movu %1, m%2
+%%wend:
+%endmacro
+
+%macro SAVE 2 ; dst, src
+ cmp pixel_maxd, (1 << 8) - 1
+ jne %%save_16bpc
+ SAVE_8BPC %1, %2
+ jmp %%end
+%%save_16bpc:
+ SAVE_16BPC %1, %2
+%%end:
+%endmacro
+
+; [rsp + even * mmsize] are gradient_h[0] - gradient_h[1]
+; [rsp + odd * mmsize] are gradient_v[0] - gradient_v[1]
+%macro APPLY_BDOF_MIN_BLOCK 4 ; block_num, vx, vy, bd
+ pxor m9, m9
+
+ movd xm10, pixel_maxd
+ vpbroadcastw m10, xm10
+
+ lea tmp0d, [pixel_maxd + 1]
+ movd xm11, tmp0d
+ VPBROADCASTW m11, xm11 ;shift_4 for pmulhrsw
+
+ APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 0
+ SAVE [dstq + 0 * dsq], 6
+
+ APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 1
+ SAVE [dstq + 1 * dsq], 6
+
+ APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 2
+ SAVE [dstq + 2 * dsq], 6
+
+ APPLY_BDOF_MIN_BLOCK_LINE m6, %2, %3, m7, (%1) * 4 + 3
+ SAVE [dstq + ds3q], 6
+%endmacro
+
+%macro SUM_MIN_BLOCK_W16 4 ; src/dst, shuffle, perm, tmp
+ pshufb %4, %1, %2
+ vpermd %4, %3, %4
+ paddw %1, %4
+%endmacro
+
+%macro SUM_MIN_BLOCK_W8 3 ; src/dst, shuffle, tmp
+ pshufb %3, %1, %2
+ paddw %1, %3
+%endmacro
+
+%macro BDOF_PROF_GRAD 2 ; line_no, last_line
+%assign i0 (%1 + 0) % 3
+%assign j0 (%1 + 1) % 3
+%assign k0 (%1 + 2) % 3
+%assign i1 3 + (%1 + 0) % 3
+%assign j1 3 + (%1 + 1) % 3
+%assign k1 3 + (%1 + 2) % 3
+
+; we cached src0 in m0 to m2
+%define t0 m %+ i0
+%define c0 m %+ j0
+%define b0 m %+ k0
+
+; we cached src1 in m3 to m5
+%define t1 m %+ i1
+%define c1 m %+ j1
+%define b1 m %+ k1
+%define ndiff t1
+%define off bdof_stack_offset(%1)
+
+ movu b0, [src0q + (%1 + 2) * SRC_STRIDE + SRC_PS]
+ movu b1, [src1q + (%1 + 2) * SRC_STRIDE + SRC_PS]
+
+ ; gradient_v[0], gradient_v[1]
+ DIFF m6, b0, t0, SHIFT, t0
+ DIFF m7, b1, t1, SHIFT, t1
+
+ ; save gradient_v[0] - gradient_v[1]
+ psubw m10, m6, m7
+ mova [rsp + off + mmsize], m10
+
+ ; gradient_h[0], gradient_h[1]
+ LOAD_GRAD_H m8, src0q, (%1 + 1) * SRC_STRIDE, t0
+ LOAD_GRAD_H m9, src1q, (%1 + 1) * SRC_STRIDE, t1
+
+ ; save gradient_h[0] - gradient_h[1]
+ psubw m11, m8, m9
+ mova [rsp + off], m11
+
+ SUM_GRAD m8, m9 ; temph
+ SUM_GRAD m6, m7 ; tempv
+
+ DIFF ndiff, c1, c0, SHIFT2, t0 ; -diff
+
+ psignw m7, ndiff, m8 ; sgxdi
+ psignw m9, ndiff, m6 ; sgydi
+ psignw m10, m8, m6 ; sgxgy
+
+ pabsw m6, m6 ; sgy2
+ pabsw m8, m8 ; sgx2
+
+ ; use t0, t1 as temporary buffers
+ cmp wd, 16
+
+ je %%w16
+ mova t0, [pb_shuffle_w8]
+ SUM_MIN_BLOCK_W8 m6, t0, m11
+ SUM_MIN_BLOCK_W8 m7, t0, m11
+ SUM_MIN_BLOCK_W8 m8, t0, m11
+ SUM_MIN_BLOCK_W8 m9, t0, m11
+ SUM_MIN_BLOCK_W8 m10, t0, m11
+ jmp %%wend
+
+%%w16:
+ mova t0, [pb_shuffle_w16]
+ mova t1, [pd_perm_w16]
+ SUM_MIN_BLOCK_W16 m6, t0, t1, m11
+ SUM_MIN_BLOCK_W16 m7, t0, t1, m11
+ SUM_MIN_BLOCK_W16 m8, t0, t1, m11
+ SUM_MIN_BLOCK_W16 m9, t0, t1, m11
+ SUM_MIN_BLOCK_W16 m10, t0, t1, m11
+
+%%wend:
+ vpblendd m11, m8, m7, 10101010b
+ vpblendd m7, m8, m7, 01010101b
+ pshufd m7, m7, q2301
+ paddw m8, m7, m11 ;4 x (2sgx2, 2sgxdi)
+
+ vpblendd m11, m6, m9, 10101010b
+ vpblendd m9, m6, m9, 01010101b
+ pshufd m9, m9, q2301
+ paddw m6, m9, m11 ;4 x (2sgy2, 2sgydi)
+
+ vpblendw m11, m8, m6, 10101010b
+ vpblendw m6, m8, m6, 01010101b
+ pshuflw m6, m6, q2301
+ pshufhw m6, m6, q2301
+ paddw m8, m6, m11 ; 4 x (4sgx2, 4sgy2, 4sgxdi, 4sgydi)
+
+%if (%1) == 0 || (%2)
+ ; pad for top and bottom
+ paddw m8, m8
+ paddw m10, m10
+%endif
+
+ paddw m12, m8
+ paddw m13, m10
+%endmacro
+
+
+%macro LOG2 5 ; log_sum, src, cmp, shift, tmp
+ pcmpgtw %5, %2, %3
+ pandd %5, %4
+ paddw %1, %5
+
+ psrlw %2, %5
+ psrlw %4, 1
+ psrlw %3, %4
+%endmacro
+
+%macro LOG2 2 ; dst/src, offset
+ pextrw tmp0d, xm%1, %2
+ bsr tmp0d, tmp0d
+ pinsrw xm%1, tmp0d, %2
+%endmacro
+
+%macro LOG2 1 ; dst/src
+ LOG2 %1, 0
+ LOG2 %1, 1
+ LOG2 %1, 2
+ LOG2 %1, 3
+ LOG2 %1, 4
+ LOG2 %1, 5
+ LOG2 %1, 6
+ LOG2 %1, 7
+%endmacro
+
+; %1: 4 (sgx2, sgy2, sgxdi, gydi)
+; %2: 4 (4sgxgy)
+%macro BDOF_VX_VY 2 ;
+ pshufd m6, m%1, q0032
+ punpckldq m%1, m6
+ vextracti128 xm7, m%1, 1
+
+ punpcklqdq m8, m%1, m7 ; 4 (sgx2, sgy2)
+ punpckhqdq m9, m%1, m7 ; 4 (sgxdi, sgydi)
+ mova m10, m8
+ LOG2 10 ; 4 (log2(sgx2), log2(sgy2))
+
+ ; Promote to dword since vpsrlvw is AVX-512 only
+ pmovsxwd m8, xm8
+ pmovsxwd m9, xm9
+ pmovsxwd m10, xm10
+
+ pslld m9, 2 ; 4 (log2(sgx2) << 2, log2(sgy2) << 2)
+
+ psignd m11, m9, m8
+ vpsravd m11, m11, m10
+ CLIPD m11, [pd_m15], [pd_15] ; 4 (vx, junk)
+
+ pshuflw m%1, m11, q0000
+ pshufhw m%1, m%1, q0000 ; 4 (2junk, 2vx)
+
+ psllq m6, m%2, 32
+ paddw m%2, m6
+
+ pmaddwd m%2, m%1 ; 4 (junk, vx * sgxgy)
+ psrad m%2, 1
+ psubd m9, m%2 ; 4 (junk, (sgydi << 2) - (vx * sgxgy >> 1))
+
+ psignd m9, m8
+ vpsravd m%2, m9, m10
+ CLIPD m%2, [pd_m15], [pd_15] ; 4 (junk, vy)
+
+ pshuflw m%2, m%2, q2222
+ pshufhw m%2, m%2, q2222 ; 4 (4vy)
+%endmacro
+
+
+%macro BDOF_MINI_BLOCKS 2 ; (block_num, last_block)
+
+%if (%1) == 0
+ movu m0, [src0q + 0 * SRC_STRIDE + SRC_PS]
+ movu m1, [src0q + 1 * SRC_STRIDE + SRC_PS]
+ movu m3, [src1q + 0 * SRC_STRIDE + SRC_PS]
+ movu m4, [src1q + 1 * SRC_STRIDE + SRC_PS]
+
+ pxor m12, m12
+ pxor m13, m13
+
+ BDOF_PROF_GRAD 0, 0
+%endif
+
+ mova m14, m12
+ mova m15, m13
+
+ pxor m12, m12
+ pxor m13, m13
+ BDOF_PROF_GRAD %1 * 4 + 1, 0
+ BDOF_PROF_GRAD %1 * 4 + 2, 0
+ paddw m14, m12
+ paddw m15, m13
+
+ pxor m12, m12
+ pxor m13, m13
+ BDOF_PROF_GRAD %1 * 4 + 3, %2
+%if (%2) == 0
+ BDOF_PROF_GRAD %1 * 4 + 4, 0
+%endif
+ paddw m14, m12
+ paddw m15, m13
+
+ BDOF_VX_VY 14, 15
+ APPLY_BDOF_MIN_BLOCK %1, m14, m15, bd
+ lea dstq, [dstq + 4 * dsq]
+%endmacro
+
+;void ff_vvc_apply_bdof_%1(uint8_t *dst, const ptrdiff_t dst_stride, int16_t *src0, int16_t *src1,
+; const int w, const int h, const int int pixel_max)
+%macro BDOF_AVX2 0
+cglobal vvc_apply_bdof, 7, 10, 16, BDOF_STACK_SIZE*32, dst, ds, src0, src1, w, h, pixel_max, ds3, tmp0, tmp1
+
+ lea ds3q, [dsq * 3]
+ sub src0q, SRC_STRIDE + SRC_PS
+ sub src1q, SRC_STRIDE + SRC_PS
+
+ BDOF_MINI_BLOCKS 0, 0
+
+ cmp hd, 16
+ je .h16
+ BDOF_MINI_BLOCKS 1, 1
+ jmp .end
+
+.h16:
+ BDOF_MINI_BLOCKS 1, 0
+ BDOF_MINI_BLOCKS 2, 0
+ BDOF_MINI_BLOCKS 3, 1
+
+.end:
+ RET
+%endmacro
+
+%macro VVC_OF_AVX2 0
+ BDOF_AVX2
+%endmacro
+
+VVC_OF_AVX2
+
+%endif ; HAVE_AVX2_EXTERNAL
+
+%endif ; ARCH_X86_64
@@ -102,6 +102,20 @@ DMVR_PROTOTYPES( 8, avx2)
DMVR_PROTOTYPES(10, avx2)
DMVR_PROTOTYPES(12, avx2)
+void ff_vvc_apply_bdof_avx2(uint8_t *dst, ptrdiff_t dst_stride, \
+ const int16_t *src0, const int16_t *src1, int w, int h, int pixel_max); \
+
+#define OF_PROTOTYPES(bd, opt) \
+static void ff_vvc_apply_bdof_##bd##_##opt(uint8_t *dst, ptrdiff_t dst_stride, \
+ const int16_t *src0, const int16_t *src1, int w, int h) \
+{ \
+ ff_vvc_apply_bdof##_##opt(dst, dst_stride, src0, src1, w, h, (1 << bd) - 1); \
+} \
+
+OF_PROTOTYPES( 8, avx2)
+OF_PROTOTYPES(10, avx2)
+OF_PROTOTYPES(12, avx2)
+
#define ALF_BPC_PROTOTYPES(bpc, opt) \
void BF(ff_vvc_alf_filter_luma, bpc, opt)(uint8_t *dst, ptrdiff_t dst_stride, \
const uint8_t *src, ptrdiff_t src_stride, ptrdiff_t width, ptrdiff_t height, \
@@ -328,6 +342,10 @@ ALF_FUNCS(16, 12, avx2)
c->inter.dmvr[1][1] = ff_vvc_dmvr_hv_##bd##_avx2; \
} while (0)
+#define OF_INIT(bd) do { \
+ c->inter.apply_bdof = ff_vvc_apply_bdof_##bd##_avx2; \
+} while (0)
+
#define ALF_INIT(bd) do { \
c->alf.filter[LUMA] = ff_vvc_alf_filter_luma_##bd##_avx2; \
c->alf.filter[CHROMA] = ff_vvc_alf_filter_chroma_##bd##_avx2; \
@@ -352,6 +370,7 @@ void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd)
ALF_INIT(8);
AVG_INIT(8, avx2);
MC_LINKS_AVX2(8);
+ OF_INIT(8);
DMVR_INIT(8);
SAD_INIT();
}
@@ -365,6 +384,7 @@ void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd)
AVG_INIT(10, avx2);
MC_LINKS_AVX2(10);
MC_LINKS_16BPC_AVX2(10);
+ OF_INIT(10);
DMVR_INIT(10);
SAD_INIT();
}
@@ -378,6 +398,7 @@ void ff_vvc_dsp_init_x86(VVCDSPContext *const c, const int bd)
AVG_INIT(12, avx2);
MC_LINKS_AVX2(12);
MC_LINKS_16BPC_AVX2(12);
+ OF_INIT(12);
DMVR_INIT(12);
SAD_INIT();
}