diff mbox

[FFmpeg-devel,v2] fate: add tests for psnr and ssim filter

Message ID 1504087412-14006-1-git-send-email-t.rapp@noa-archive.com
State Superseded
Headers show

Commit Message

Tobias Rapp Aug. 30, 2017, 10:03 a.m. UTC 
Metadata output is passed through an Awk script reducing precision of
floats to account for architecture differences (e.g x86-32 vs. x86-64).

Signed-off-by: Tobias Rapp <t.rapp@noa-archive.com>
---
v2:
 - removed CPUFLAGS work-around for ssim filter issue
 - added metadata float value post-processing script

 tests/fate-run.sh                     |  9 +++++++
 tests/fate/filter-video.mak           | 14 +++++++++++
 tests/ref/fate/filter-refcmp-psnr-rgb | 45 +++++++++++++++++++++++++++++++++++
 tests/ref/fate/filter-refcmp-psnr-yuv | 45 +++++++++++++++++++++++++++++++++++
 tests/ref/fate/filter-refcmp-ssim-rgb | 30 +++++++++++++++++++++++
 tests/ref/fate/filter-refcmp-ssim-yuv | 30 +++++++++++++++++++++++
 tests/refcmp-metadata.awk             | 41 +++++++++++++++++++++++++++++++
 7 files changed, 214 insertions(+)
 create mode 100644 tests/ref/fate/filter-refcmp-psnr-rgb
 create mode 100644 tests/ref/fate/filter-refcmp-psnr-yuv
 create mode 100644 tests/ref/fate/filter-refcmp-ssim-rgb
 create mode 100644 tests/ref/fate/filter-refcmp-ssim-yuv
 create mode 100644 tests/refcmp-metadata.awk

Comments

Nicolas George Aug. 30, 2017, 10:08 a.m. UTC | #1 
Le tridi 13 fructidor, an CCXXV, Tobias Rapp a écrit :
> Metadata output is passed through an Awk script reducing precision of
> floats to account for architecture differences (e.g x86-32 vs. x86-64).

I do not like that approach at all: it will miss small differences
introduced inadvertently by changes, making the test less useful, but
would detect falsely rounding differences that fall near the cutoff
(0.499 -> 0.501).

I had posted I script to make approximate comparisons of text files; I
still like this approach better.

Regards,
Tobias Rapp Aug. 30, 2017, 10:47 a.m. UTC | #2 
On 30.08.2017 12:08, Nicolas George wrote:
> Le tridi 13 fructidor, an CCXXV, Tobias Rapp a écrit :
>> Metadata output is passed through an Awk script reducing precision of
>> floats to account for architecture differences (e.g x86-32 vs. x86-64).
> 
> I do not like that approach at all: it will miss small differences
> introduced inadvertently by changes, making the test less useful, but
> would detect falsely rounding differences that fall near the cutoff
> (0.499 -> 0.501).
> 
> I had posted I script to make approximate comparisons of text files; I
> still like this approach better.

It is true that the "precision" parameter of your approach can be half 
the "fuzz" parameter of my approach. Also the "precision" parameter 
would be defined per float value while this patch uses a normalized 
"fuzz" parameter that is scaled depending on the float value's exponent.

There are three considerations that made me choose the current path of 
implementation:

a) With the proposed patch it is possible to generate/update a reference 
file simply by using "make fate GEN=1".

b) Adding a test with slightly more tolerance than needed is better than 
having no test at all.

c) As far as I can see Perl is currently not a dependency for "make 
fate" and I didn't want to add it.

Regards,
Tobias
diff mbox

Patch

diff --git a/tests/fate-run.sh b/tests/fate-run.sh
index 9aa9a22..5af9a36 100755
--- a/tests/fate-run.sh
+++ b/tests/fate-run.sh
@@ -234,6 +234,15 @@  lavftest(){
     ${base}/lavf-regression.sh $t lavf tests/vsynth1 "$target_exec" "$target_path" "$threads" "$thread_type" "$cpuflags" "$target_samples"
 }
 
+refcmp_metadata(){
+    refcmp=$1
+    pixfmt=$2
+    fuzz=${3:-0.01}
+    ffmpeg $FLAGS $ENC_OPTS \
+        -lavfi "testsrc2=size=300x200:rate=1:duration=5,format=${pixfmt},split[ref][tmp];[tmp]avgblur=4[enc];[enc][ref]${refcmp},metadata=print:file=-" \
+        -f null /dev/null | awk -v fuzz=${fuzz} -f ${base}/refcmp-metadata.awk -
+}
+
 video_filter(){
     filters=$1
     shift
diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak
index 670d9de..9f2fca7 100644
--- a/tests/fate/filter-video.mak
+++ b/tests/fate/filter-video.mak
@@ -717,6 +717,20 @@  FATE_FILTER_SAMPLES-$(call ALLYES, MOV_DEMUXER H264_DECODER AAC_FIXED_DECODER PC
 fate-filter-meta-4560-rotate0: tests/data/file4560-override2rotate0.mov
 fate-filter-meta-4560-rotate0: CMD = framecrc -flags +bitexact -c:a aac_fixed -i $(TARGET_PATH)/tests/data/file4560-override2rotate0.mov
 
+REFCMP_DEPS = FFMPEG LAVFI_INDEV TESTSRC2_FILTER AVGBLUR_FILTER METADATA_FILTER
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-rgb
+fate-filter-refcmp-psnr-rgb: CMD = refcmp_metadata psnr rgb24
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) PSNR_FILTER) += fate-filter-refcmp-psnr-yuv
+fate-filter-refcmp-psnr-yuv: CMD = refcmp_metadata psnr yuv422p
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-rgb
+fate-filter-refcmp-ssim-rgb: CMD = refcmp_metadata ssim rgb24
+
+FATE_FILTER_SAMPLES-$(call ALLYES, $(REFCMP_DEPS) SSIM_FILTER) += fate-filter-refcmp-ssim-yuv
+fate-filter-refcmp-ssim-yuv: CMD = refcmp_metadata ssim yuv422p
+
 FATE_SAMPLES_FFPROBE += $(FATE_METADATA_FILTER-yes)
 FATE_SAMPLES_FFMPEG += $(FATE_FILTER_SAMPLES-yes)
 FATE_FFMPEG += $(FATE_FILTER-yes)
diff --git a/tests/ref/fate/filter-refcmp-psnr-rgb b/tests/ref/fate/filter-refcmp-psnr-rgb
new file mode 100644
index 0000000..16b2208
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-psnr-rgb
@@ -0,0 +1,45 @@ 
+frame:0    pts:0       pts_time:0      
+lavfi.psnr.mse.r=1380.00
+lavfi.psnr.psnr.r=16.70
+lavfi.psnr.mse.g=896.00
+lavfi.psnr.psnr.g=18.60
+lavfi.psnr.mse.b=277.00
+lavfi.psnr.psnr.b=23.70
+lavfi.psnr.mse_avg=852.00
+lavfi.psnr.psnr_avg=18.80
+frame:1    pts:1       pts_time:1      
+lavfi.psnr.mse.r=1380.00
+lavfi.psnr.psnr.r=16.70
+lavfi.psnr.mse.g=976.00
+lavfi.psnr.psnr.g=18.20
+lavfi.psnr.mse.b=436.00
+lavfi.psnr.psnr.b=21.70
+lavfi.psnr.mse_avg=931.00
+lavfi.psnr.psnr_avg=18.40
+frame:2    pts:2       pts_time:2      
+lavfi.psnr.mse.r=1400.00
+lavfi.psnr.psnr.r=16.70
+lavfi.psnr.mse.g=954.00
+lavfi.psnr.psnr.g=18.30
+lavfi.psnr.mse.b=494.00
+lavfi.psnr.psnr.b=21.20
+lavfi.psnr.mse_avg=950.00
+lavfi.psnr.psnr_avg=18.40
+frame:3    pts:3       pts_time:3      
+lavfi.psnr.mse.r=1450.00
+lavfi.psnr.psnr.r=16.50
+lavfi.psnr.mse.g=1000.00
+lavfi.psnr.psnr.g=18.10
+lavfi.psnr.mse.b=557.00
+lavfi.psnr.psnr.b=20.70
+lavfi.psnr.mse_avg=1000.00
+lavfi.psnr.psnr_avg=18.10
+frame:4    pts:4       pts_time:4      
+lavfi.psnr.mse.r=1400.00
+lavfi.psnr.psnr.r=16.70
+lavfi.psnr.mse.g=1010.00
+lavfi.psnr.psnr.g=18.10
+lavfi.psnr.mse.b=602.00
+lavfi.psnr.psnr.b=20.30
+lavfi.psnr.mse_avg=1000.00
+lavfi.psnr.psnr_avg=18.10
diff --git a/tests/ref/fate/filter-refcmp-psnr-yuv b/tests/ref/fate/filter-refcmp-psnr-yuv
new file mode 100644
index 0000000..b84c156
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-psnr-yuv
@@ -0,0 +1,45 @@ 
+frame:0    pts:0       pts_time:0      
+lavfi.psnr.mse.y=222.00
+lavfi.psnr.psnr.y=24.70
+lavfi.psnr.mse.u=339.00
+lavfi.psnr.psnr.u=22.80
+lavfi.psnr.mse.v=705.00
+lavfi.psnr.psnr.v=19.60
+lavfi.psnr.mse_avg=372.00
+lavfi.psnr.psnr_avg=22.40
+frame:1    pts:1       pts_time:1      
+lavfi.psnr.mse.y=237.00
+lavfi.psnr.psnr.y=24.40
+lavfi.psnr.mse.u=416.00
+lavfi.psnr.psnr.u=21.90
+lavfi.psnr.mse.v=705.00
+lavfi.psnr.psnr.v=19.60
+lavfi.psnr.mse_avg=399.00
+lavfi.psnr.psnr_avg=22.10
+frame:2    pts:2       pts_time:2      
+lavfi.psnr.mse.y=235.00
+lavfi.psnr.psnr.y=24.40
+lavfi.psnr.mse.u=436.00
+lavfi.psnr.psnr.u=21.70
+lavfi.psnr.mse.v=700.00
+lavfi.psnr.psnr.v=19.70
+lavfi.psnr.mse_avg=401.00
+lavfi.psnr.psnr_avg=22.10
+frame:3    pts:3       pts_time:3      
+lavfi.psnr.mse.y=251.00
+lavfi.psnr.psnr.y=24.10
+lavfi.psnr.mse.u=480.00
+lavfi.psnr.psnr.u=21.30
+lavfi.psnr.mse.v=708.00
+lavfi.psnr.psnr.v=19.60
+lavfi.psnr.mse_avg=422.00
+lavfi.psnr.psnr_avg=21.90
+frame:4    pts:4       pts_time:4      
+lavfi.psnr.mse.y=241.00
+lavfi.psnr.psnr.y=24.30
+lavfi.psnr.mse.u=505.00
+lavfi.psnr.psnr.u=21.10
+lavfi.psnr.mse.v=716.00
+lavfi.psnr.psnr.v=19.60
+lavfi.psnr.mse_avg=426.00
+lavfi.psnr.psnr_avg=21.80
diff --git a/tests/ref/fate/filter-refcmp-ssim-rgb b/tests/ref/fate/filter-refcmp-ssim-rgb
new file mode 100644
index 0000000..4a06eb4
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-ssim-rgb
@@ -0,0 +1,30 @@ 
+frame:0    pts:0       pts_time:0      
+lavfi.ssim.R=0.72
+lavfi.ssim.G=0.76
+lavfi.ssim.B=0.89
+lavfi.ssim.All=0.79
+lavfi.ssim.dB=6.74
+frame:1    pts:1       pts_time:1      
+lavfi.ssim.R=0.70
+lavfi.ssim.G=0.74
+lavfi.ssim.B=0.85
+lavfi.ssim.All=0.77
+lavfi.ssim.dB=6.31
+frame:2    pts:2       pts_time:2      
+lavfi.ssim.R=0.71
+lavfi.ssim.G=0.75
+lavfi.ssim.B=0.84
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.29
+frame:3    pts:3       pts_time:3      
+lavfi.ssim.R=0.70
+lavfi.ssim.G=0.73
+lavfi.ssim.B=0.83
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.11
+frame:4    pts:4       pts_time:4      
+lavfi.ssim.R=0.71
+lavfi.ssim.G=0.74
+lavfi.ssim.B=0.80
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.05
diff --git a/tests/ref/fate/filter-refcmp-ssim-yuv b/tests/ref/fate/filter-refcmp-ssim-yuv
new file mode 100644
index 0000000..0cd46c8
--- /dev/null
+++ b/tests/ref/fate/filter-refcmp-ssim-yuv
@@ -0,0 +1,30 @@ 
+frame:0    pts:0       pts_time:0      
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.76
+lavfi.ssim.V=0.69
+lavfi.ssim.All=0.76
+lavfi.ssim.dB=6.25
+frame:1    pts:1       pts_time:1      
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.73
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.08
+frame:2    pts:2       pts_time:2      
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.73
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=6.10
+frame:3    pts:3       pts_time:3      
+lavfi.ssim.Y=0.79
+lavfi.ssim.U=0.72
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=5.94
+frame:4    pts:4       pts_time:4      
+lavfi.ssim.Y=0.80
+lavfi.ssim.U=0.72
+lavfi.ssim.V=0.68
+lavfi.ssim.All=0.75
+lavfi.ssim.dB=5.97
diff --git a/tests/refcmp-metadata.awk b/tests/refcmp-metadata.awk
new file mode 100644
index 0000000..a4d37f6
--- /dev/null
+++ b/tests/refcmp-metadata.awk
@@ -0,0 +1,41 @@ 
+# normalize metadata filter output containing float value strings
+
+BEGIN {
+    FS = "=";
+    if (fuzz <= 0.0) {
+        fuzz = 0.1;
+    }
+}
+
+function floor(val) {
+    ival = int(val);
+    if (val == ival) {
+        return ival;
+    } else if (val < 0) {
+        return ival - 1;
+    } else {
+        return ival;
+    }
+}
+
+function round(val, fuzz) {
+    if (val == 0) {
+        return 0;
+    } else if (val < 0) {
+        sign = -1;
+        val = -val;
+    } else {
+        sign = +1;
+    }
+    norm = fuzz * 10^floor(log(val) / log(10));
+    return int(val / norm + 0.5) * norm * sign;
+}
+
+{
+    if (NF == 2 && $2 ~ /^[+-]?[0-9]*\.?[0-9]+$/) {
+        val = $2 + 0;  # convert to number
+        printf "%s=%.2f\n", $1, round(val, fuzz);
+    } else {
+        print $0;
+    }
+}