diff mbox series

[FFmpeg-devel] tests/fate: add simple tests for libx264 and libx265

Message ID 20230311170502.3451-1-anton@khirnov.net
State New
Headers show
Series [FFmpeg-devel] tests/fate: add simple tests for libx264 and libx265 | expand

Checks

Context Check Description
yinshiyou/make_loongarch64 success Make finished
yinshiyou/make_fate_loongarch64 success Make fate finished
andriy/make_x86 success Make finished
andriy/make_fate_x86 success Make fate finished

Commit Message

Anton Khirnov March 11, 2023, 5:05 p.m. UTC 
Since these are external encoders not under our control, we cannot test
the encoded output exactly as is done for internal encoders. We can
still test however that the output is decodable and produces the
expected number of frames with expected dimensions, pixel formats, and
timestamps.
---
A minimal proof of concept, more tests for these and other external
encoders are welcome.

Samples were picked at a whim, better suggestions also welcome.
---
 tests/Makefile                |  1 +
 tests/fate-run.sh             | 16 +++++++
 tests/fate/enc_external.mak   | 10 +++++
 tests/ref/fate/libx264-simple | 85 +++++++++++++++++++++++++++++++++++
 tests/ref/fate/libx265-simple | 25 +++++++++++
 5 files changed, 137 insertions(+)
 create mode 100644 tests/fate/enc_external.mak
 create mode 100644 tests/ref/fate/libx264-simple
 create mode 100644 tests/ref/fate/libx265-simple

Comments

Tobias Rapp March 14, 2023, 8:29 a.m. UTC | #1 
On 11/03/2023 18:05, Anton Khirnov wrote:

> Since these are external encoders not under our control, we cannot test
> the encoded output exactly as is done for internal encoders. We can
> still test however that the output is decodable and produces the
> expected number of frames with expected dimensions, pixel formats, and
> timestamps.
> ---
> A minimal proof of concept, more tests for these and other external
> encoders are welcome.
>
> Samples were picked at a whim, better suggestions also welcome.
> ---
>   tests/Makefile                |  1 +
>   tests/fate-run.sh             | 16 +++++++
>   tests/fate/enc_external.mak   | 10 +++++
>   tests/ref/fate/libx264-simple | 85 +++++++++++++++++++++++++++++++++++
>   tests/ref/fate/libx265-simple | 25 +++++++++++
>   5 files changed, 137 insertions(+)
>   create mode 100644 tests/fate/enc_external.mak
>   create mode 100644 tests/ref/fate/libx264-simple
>   create mode 100644 tests/ref/fate/libx265-simple
> [...]

I think your approach for testing external (video) encoders is meaningful.

Regards, Tobias
Anton Khirnov May 15, 2023, 8:51 a.m. UTC | #2 
Since libx265 has issues (leaks, invalid reads), and I don't have time
to craft supressions files right now, I pushed just the x264 part of
this patch for now.
diff mbox series

Patch

diff --git a/tests/Makefile b/tests/Makefile
index 1d50e1d175..8071bd829d 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -178,6 +178,7 @@  include $(SRC_PATH)/tests/fate/dpcm.mak
 include $(SRC_PATH)/tests/fate/dvvideo.mak
 include $(SRC_PATH)/tests/fate/ea.mak
 include $(SRC_PATH)/tests/fate/exif.mak
+include $(SRC_PATH)/tests/fate/enc_external.mak
 # Must be included after lavf-video.mak
 include $(SRC_PATH)/tests/fate/ffmpeg.mak
 include $(SRC_PATH)/tests/fate/ffprobe.mak
diff --git a/tests/fate-run.sh b/tests/fate-run.sh
index 61cc59acc0..e12279e4cf 100755
--- a/tests/fate-run.sh
+++ b/tests/fate-run.sh
@@ -282,6 +282,22 @@  stream_remux(){
         run ffprobe${PROGSUF}${EXECSUF} -bitexact $ffprobe_opts $tencfile || return
 }
 
+# this function is for testing external encoders,
+# where the precise output is not controlled by us
+# we can still test e.g. that the output can be decoded correctly
+enc_external(){
+    srcfile=$1
+    enc_fmt=$2
+    enc_opt=$3
+    probe_opt=$4
+
+    srcfile=$(target_path $srcfile)
+    encfile=$(target_path "${outdir}/${test}.${enc_fmt}")
+
+    ffmpeg -i $srcfile $enc_opt -f $enc_fmt -y $encfile || return
+    run ffprobe${PROGSUF}${EXECSUF} -bitexact $probe_opt $encfile || return
+}
+
 # FIXME: There is a certain duplication between the avconv-related helper
 # functions above and below that should be refactored.
 ffmpeg2="$target_exec ${target_path}/ffmpeg${PROGSUF}${EXECSUF}"
diff --git a/tests/fate/enc_external.mak b/tests/fate/enc_external.mak
new file mode 100644
index 0000000000..6492b52fa1
--- /dev/null
+++ b/tests/fate/enc_external.mak
@@ -0,0 +1,10 @@ 
+FATE_ENC_EXTERNAL-$(call ENCDEC, LIBX264 H264, MOV, H264_DEMUXER) += fate-libx264-simple
+fate-libx264-simple: CMD = enc_external $(TARGET_SAMPLES)/h264-conformance/BA1_Sony_D.jsv \
+    mp4 "-c:v libx264" "-show_entries frame=width,height,pix_fmt,pts,pkt_dts -of flat"
+
+FATE_ENC_EXTERNAL-$(call ENCDEC, LIBX265 HEVC, MOV, HEVC_DEMUXER) += fate-libx265-simple
+fate-libx265-simple: CMD = enc_external $(TARGET_SAMPLES)/hevc-conformance/POC_A_Bossen_3.bit \
+    mp4 "-c:v libx265" "-show_entries frame=width,height,pix_fmt,pts,pkt_dts -of flat"
+
+FATE_SAMPLES_FFMPEG_FFPROBE += $(FATE_ENC_EXTERNAL-yes)
+fate-enc-external: $(FATE_ENC_EXTERNAL-yes)
diff --git a/tests/ref/fate/libx264-simple b/tests/ref/fate/libx264-simple
new file mode 100644
index 0000000000..b7f7e67c3b
--- /dev/null
+++ b/tests/ref/fate/libx264-simple
@@ -0,0 +1,85 @@ 
+frames.frame.0.pts=0
+frames.frame.0.pkt_dts=0
+frames.frame.0.width=176
+frames.frame.0.height=144
+frames.frame.0.pix_fmt="yuv420p"
+frames.frame.1.pts=512
+frames.frame.1.pkt_dts=512
+frames.frame.1.width=176
+frames.frame.1.height=144
+frames.frame.1.pix_fmt="yuv420p"
+frames.frame.2.pts=1024
+frames.frame.2.pkt_dts=1024
+frames.frame.2.width=176
+frames.frame.2.height=144
+frames.frame.2.pix_fmt="yuv420p"
+frames.frame.3.pts=1536
+frames.frame.3.pkt_dts=1536
+frames.frame.3.width=176
+frames.frame.3.height=144
+frames.frame.3.pix_fmt="yuv420p"
+frames.frame.4.pts=2048
+frames.frame.4.pkt_dts=2048
+frames.frame.4.width=176
+frames.frame.4.height=144
+frames.frame.4.pix_fmt="yuv420p"
+frames.frame.5.pts=2560
+frames.frame.5.pkt_dts=2560
+frames.frame.5.width=176
+frames.frame.5.height=144
+frames.frame.5.pix_fmt="yuv420p"
+frames.frame.6.pts=3072
+frames.frame.6.pkt_dts=3072
+frames.frame.6.width=176
+frames.frame.6.height=144
+frames.frame.6.pix_fmt="yuv420p"
+frames.frame.7.pts=3584
+frames.frame.7.pkt_dts=3584
+frames.frame.7.width=176
+frames.frame.7.height=144
+frames.frame.7.pix_fmt="yuv420p"
+frames.frame.8.pts=4096
+frames.frame.8.pkt_dts=4096
+frames.frame.8.width=176
+frames.frame.8.height=144
+frames.frame.8.pix_fmt="yuv420p"
+frames.frame.9.pts=4608
+frames.frame.9.pkt_dts=4608
+frames.frame.9.width=176
+frames.frame.9.height=144
+frames.frame.9.pix_fmt="yuv420p"
+frames.frame.10.pts=5120
+frames.frame.10.pkt_dts=5120
+frames.frame.10.width=176
+frames.frame.10.height=144
+frames.frame.10.pix_fmt="yuv420p"
+frames.frame.11.pts=5632
+frames.frame.11.pkt_dts=5632
+frames.frame.11.width=176
+frames.frame.11.height=144
+frames.frame.11.pix_fmt="yuv420p"
+frames.frame.12.pts=6144
+frames.frame.12.pkt_dts=6144
+frames.frame.12.width=176
+frames.frame.12.height=144
+frames.frame.12.pix_fmt="yuv420p"
+frames.frame.13.pts=6656
+frames.frame.13.pkt_dts=6656
+frames.frame.13.width=176
+frames.frame.13.height=144
+frames.frame.13.pix_fmt="yuv420p"
+frames.frame.14.pts=7168
+frames.frame.14.pkt_dts=7168
+frames.frame.14.width=176
+frames.frame.14.height=144
+frames.frame.14.pix_fmt="yuv420p"
+frames.frame.15.pts=7680
+frames.frame.15.pkt_dts="N/A"
+frames.frame.15.width=176
+frames.frame.15.height=144
+frames.frame.15.pix_fmt="yuv420p"
+frames.frame.16.pts=8192
+frames.frame.16.pkt_dts="N/A"
+frames.frame.16.width=176
+frames.frame.16.height=144
+frames.frame.16.pix_fmt="yuv420p"
diff --git a/tests/ref/fate/libx265-simple b/tests/ref/fate/libx265-simple
new file mode 100644
index 0000000000..dffe4bcb0e
--- /dev/null
+++ b/tests/ref/fate/libx265-simple
@@ -0,0 +1,25 @@ 
+frames.frame.0.pts=0
+frames.frame.0.pkt_dts=0
+frames.frame.0.width=416
+frames.frame.0.height=240
+frames.frame.0.pix_fmt="yuv420p"
+frames.frame.1.pts=512
+frames.frame.1.pkt_dts=512
+frames.frame.1.width=416
+frames.frame.1.height=240
+frames.frame.1.pix_fmt="yuv420p"
+frames.frame.2.pts=1024
+frames.frame.2.pkt_dts=1024
+frames.frame.2.width=416
+frames.frame.2.height=240
+frames.frame.2.pix_fmt="yuv420p"
+frames.frame.3.pts=1536
+frames.frame.3.pkt_dts="N/A"
+frames.frame.3.width=416
+frames.frame.3.height=240
+frames.frame.3.pix_fmt="yuv420p"
+frames.frame.4.pts=2048
+frames.frame.4.pkt_dts="N/A"
+frames.frame.4.width=416
+frames.frame.4.height=240
+frames.frame.4.pix_fmt="yuv420p"