[FFmpeg-devel,2/2] fate: use do_md5sum instead of the md5 protocol for md5 fate tests

Submitted by Marton Balint on June 16, 2017, 11:24 p.m.

Details

Message ID 20170616232442.20537-2-cus@passwd.hu
State New
Headers show

Commit Message

Marton Balint June 16, 2017, 11:24 p.m.
The md5 protocol has no seek support, but some tests use seeks. This changes
the fate tests to actually create the output files and calculate the md5 on the
written files, which also makes the tests independent of the size of the output
buffers and output buffering in general.

Signed-off-by: Marton Balint <cus@passwd.hu>
---
 tests/fate-run.sh                                 | 5 ++++-
 tests/fate/filter-audio.mak                       | 4 ++--
 tests/fate/matroska.mak                           | 2 +-
 tests/fate/wavpack.mak                            | 4 ++--
 tests/ref/fate/binsub-mksenc                      | 2 +-
 tests/ref/fate/mapchan-6ch-extract-2-downmix-mono | 2 +-
 tests/ref/fate/mapchan-silent-mono                | 2 +-
 tests/ref/fate/mpeg4-bsf-unpack-bframes           | 2 +-
 tests/ref/fate/time_base                          | 2 +-
 tests/ref/fate/v410enc                            | 2 +-
 10 files changed, 15 insertions(+), 12 deletions(-)

Comments

James Almer June 16, 2017, 11:32 p.m.
On 6/16/2017 8:24 PM, Marton Balint wrote:
> The md5 protocol has no seek support, but some tests use seeks. This changes
> the fate tests to actually create the output files and calculate the md5 on the
> written files, which also makes the tests independent of the size of the output
> buffers and output buffering in general.

The fact md5() forced non seekable output came in handy to test certain
codepaths, especially in matroska. Perhaps you could instead add a new
fate-run.sh function for this change and port those tests that use
output formats that don't behave any different in non seekable output?

> 
> Signed-off-by: Marton Balint <cus@passwd.hu>
> ---
>  tests/fate-run.sh                                 | 5 ++++-
>  tests/fate/filter-audio.mak                       | 4 ++--
>  tests/fate/matroska.mak                           | 2 +-
>  tests/fate/wavpack.mak                            | 4 ++--
>  tests/ref/fate/binsub-mksenc                      | 2 +-
>  tests/ref/fate/mapchan-6ch-extract-2-downmix-mono | 2 +-
>  tests/ref/fate/mapchan-silent-mono                | 2 +-
>  tests/ref/fate/mpeg4-bsf-unpack-bframes           | 2 +-
>  tests/ref/fate/time_base                          | 2 +-
>  tests/ref/fate/v410enc                            | 2 +-
>  10 files changed, 15 insertions(+), 12 deletions(-)
> 
> diff --git a/tests/fate-run.sh b/tests/fate-run.sh
> index 931150b5a6..20dae9ea13 100755
> --- a/tests/fate-run.sh
> +++ b/tests/fate-run.sh
> @@ -143,7 +143,10 @@ crc(){
>  }
>  
>  md5(){
> -    ffmpeg "$@" md5:
> +    encfile="${outdir}/${test}.out"
> +    cleanfiles=$encfile
> +    ffmpeg "$@" $encfile
> +    do_md5sum $encfile | awk '{print $1}'
>  }
>  
>  pcm(){
> diff --git a/tests/fate/filter-audio.mak b/tests/fate/filter-audio.mak
> index 5d15b31e0b..8030641ed6 100644
> --- a/tests/fate/filter-audio.mak
> +++ b/tests/fate/filter-audio.mak
> @@ -215,7 +215,7 @@ fate-filter-channelmap-one-int: SRC = $(TARGET_PATH)/tests/data/asynth-44100-6.w
>  fate-filter-channelmap-one-int: tests/data/asynth-44100-6.wav
>  fate-filter-channelmap-one-int: CMD = md5 -i $(SRC) -filter_complex_script $(TARGET_PATH)/tests/data/filtergraphs/channelmap_one_int -f wav -fflags +bitexact
>  fate-filter-channelmap-one-int: CMP = oneline
> -fate-filter-channelmap-one-int: REF = 428b8f9fac6d57147069b97335019ef5
> +fate-filter-channelmap-one-int: REF = 8cfe553d65ed4696756d8c1b824fcdd3
>  
>  FATE_FILTER_CHANNELMAP += fate-filter-channelmap-one-str
>  fate-filter-channelmap-one-str: tests/data/filtergraphs/channelmap_one_str
> @@ -223,7 +223,7 @@ fate-filter-channelmap-one-str: SRC = $(TARGET_PATH)/tests/data/asynth-44100-2.w
>  fate-filter-channelmap-one-str: tests/data/asynth-44100-2.wav
>  fate-filter-channelmap-one-str: CMD = md5 -i $(SRC) -filter_complex_script $(TARGET_PATH)/tests/data/filtergraphs/channelmap_one_str -f wav -fflags +bitexact
>  fate-filter-channelmap-one-str: CMP = oneline
> -fate-filter-channelmap-one-str: REF = e788890db6a11c2fb29d7c4229072d49
> +fate-filter-channelmap-one-str: REF = 0ea3052e482c95d5d3bd9da6dac1b5fa
>  
>  FATE_AFILTER-$(call FILTERDEMDECENCMUX, CHANNELMAP, WAV, PCM_S16LE, PCM_S16LE, WAV) += $(FATE_FILTER_CHANNELMAP)
>  
> diff --git a/tests/fate/matroska.mak b/tests/fate/matroska.mak
> index 48b710c26c..9e233cf510 100644
> --- a/tests/fate/matroska.mak
> +++ b/tests/fate/matroska.mak
> @@ -4,7 +4,7 @@
>  FATE_MATROSKA-$(call DEMMUX, MATROSKA, MATROSKA) += fate-matroska-remux
>  fate-matroska-remux: CMD = md5 -i $(TARGET_SAMPLES)/vp9-test-vectors/vp90-2-2pass-akiyo.webm -color_trc 4 -c:v copy -fflags +bitexact -strict -2 -f matroska
>  fate-matroska-remux: CMP = oneline
> -fate-matroska-remux: REF = 1ed49a4f2b6790357fac268938357353
> +fate-matroska-remux: REF = 53be067cfaee3626d1c94b4a0fd5ac10
>  
>  FATE_MATROSKA_FFPROBE-$(call ALLYES, MATROSKA_DEMUXER) += fate-matroska-spherical-mono
>  fate-matroska-spherical-mono: CMD = run ffprobe$(PROGSSUF)$(EXESUF) -show_entries stream_side_data_list -select_streams v -v 0 $(TARGET_SAMPLES)/mkv/spherical.mkv
> diff --git a/tests/fate/wavpack.mak b/tests/fate/wavpack.mak
> index 32ae3f6100..1cb414447c 100644
> --- a/tests/fate/wavpack.mak
> +++ b/tests/fate/wavpack.mak
> @@ -91,12 +91,12 @@ fate-wavpack-matroskamode: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/special/matros
>  FATE_WAVPACK-$(call DEMMUX, WV, MATROSKA) += fate-wavpack-matroska_mux-mono
>  fate-wavpack-matroska_mux-mono: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/num_channels/mono_16bit_int.wv -c copy -fflags +bitexact -f matroska
>  fate-wavpack-matroska_mux-mono: CMP = oneline
> -fate-wavpack-matroska_mux-mono: REF = 11773e2a518edc788475f3880d849230
> +fate-wavpack-matroska_mux-mono: REF = 31c0b4e71b532103bbfd2e284795cac1
>  
>  FATE_WAVPACK-$(call DEMMUX, WV, MATROSKA) += fate-wavpack-matroska_mux-61
>  fate-wavpack-matroska_mux-61: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/num_channels/eva_2.22_6.1_16bit-partial.wv -c copy -fflags +bitexact -f matroska
>  fate-wavpack-matroska_mux-61: CMP = oneline
> -fate-wavpack-matroska_mux-61: REF = 9641abdf596c10c2e21bd9b026d4bade
> +fate-wavpack-matroska_mux-61: REF = a0e5ad857d57a50869c21f1621e55674
>  
>  FATE_SAMPLES_AVCONV += $(FATE_WAVPACK-yes)
>  fate-wavpack: $(FATE_WAVPACK-yes)
> diff --git a/tests/ref/fate/binsub-mksenc b/tests/ref/fate/binsub-mksenc
> index f247d9d22d..a0a58b880d 100644
> --- a/tests/ref/fate/binsub-mksenc
> +++ b/tests/ref/fate/binsub-mksenc
> @@ -1 +1 @@
> -f80f42e646fce972e73aa6d99dcfa470
> +cdc96049a5aeaccec0126f19600d88aa
> diff --git a/tests/ref/fate/mapchan-6ch-extract-2-downmix-mono b/tests/ref/fate/mapchan-6ch-extract-2-downmix-mono
> index a47c85cd6b..9ac4dba261 100644
> --- a/tests/ref/fate/mapchan-6ch-extract-2-downmix-mono
> +++ b/tests/ref/fate/mapchan-6ch-extract-2-downmix-mono
> @@ -1 +1 @@
> -2727e5418eb5b8f89954f140d9f3a19a
> +6ededcfc9f0b26d200a7295205e87663
> diff --git a/tests/ref/fate/mapchan-silent-mono b/tests/ref/fate/mapchan-silent-mono
> index f720c318f4..93f519e0ee 100644
> --- a/tests/ref/fate/mapchan-silent-mono
> +++ b/tests/ref/fate/mapchan-silent-mono
> @@ -1 +1 @@
> -16dc73c037897dff9b1d3e3d8561dc73
> +b6b034fffa8639e64d79be84f16d9ebe
> diff --git a/tests/ref/fate/mpeg4-bsf-unpack-bframes b/tests/ref/fate/mpeg4-bsf-unpack-bframes
> index 162d436625..703ae3a8e8 100644
> --- a/tests/ref/fate/mpeg4-bsf-unpack-bframes
> +++ b/tests/ref/fate/mpeg4-bsf-unpack-bframes
> @@ -1 +1 @@
> -c9535e459c2ee4ead6d84b93bc7e9f46
> +a0c4ba21ab9d68966b8fce7cc848e2c6
> diff --git a/tests/ref/fate/time_base b/tests/ref/fate/time_base
> index eec4b21aa5..7923556b35 100644
> --- a/tests/ref/fate/time_base
> +++ b/tests/ref/fate/time_base
> @@ -1 +1 @@
> -ef9896ce045941607782e895aea930b6
> +d26a35b141551b36c5b8bd716451cfcb
> diff --git a/tests/ref/fate/v410enc b/tests/ref/fate/v410enc
> index 139da7b875..9fddf5a35c 100644
> --- a/tests/ref/fate/v410enc
> +++ b/tests/ref/fate/v410enc
> @@ -1 +1 @@
> -5fd2d9a7b3311f5c19dbdd647bb9eae6
> +465bcc7477104a8295f47b35f1b987df
>
Marton Balint June 17, 2017, 12:36 a.m.
On Fri, 16 Jun 2017, James Almer wrote:

> On 6/16/2017 8:24 PM, Marton Balint wrote:
>> The md5 protocol has no seek support, but some tests use seeks. This changes
>> the fate tests to actually create the output files and calculate the md5 on the
>> written files, which also makes the tests independent of the size of the output
>> buffers and output buffering in general.
>
> The fact md5() forced non seekable output came in handy to test certain
> codepaths, especially in matroska. Perhaps you could instead add a new
> fate-run.sh function for this change and port those tests that use
> output formats that don't behave any different in non seekable output?
>

IMHO when most people are using the md5 test, they are typically not aware 
the lack of seek support. Considering how many md5 tests are used all over 
fate, I'd rather create a test named md5pipe which uses the md5 protocol 
(this name better reflects the streaming property of the test as well), 
and change the matroska tests to use that, if you really want to test the 
streaming output.

BTW in the matroska muxer, should a non-seekable output and using the 
-live option produce the same result? Because there are some places in the 
code which only checks if -live is set, and I am not sure if that is 
intentional.

Regards,
Marton
James Almer June 17, 2017, 12:41 a.m.
On 6/16/2017 9:36 PM, Marton Balint wrote:
> 
> On Fri, 16 Jun 2017, James Almer wrote:
> 
>> On 6/16/2017 8:24 PM, Marton Balint wrote:
>>> The md5 protocol has no seek support, but some tests use seeks. This
>>> changes
>>> the fate tests to actually create the output files and calculate the
>>> md5 on the
>>> written files, which also makes the tests independent of the size of
>>> the output
>>> buffers and output buffering in general.
>>
>> The fact md5() forced non seekable output came in handy to test certain
>> codepaths, especially in matroska. Perhaps you could instead add a new
>> fate-run.sh function for this change and port those tests that use
>> output formats that don't behave any different in non seekable output?
>>
> 
> IMHO when most people are using the md5 test, they are typically not
> aware the lack of seek support. Considering how many md5 tests are used
> all over fate, I'd rather create a test named md5pipe which uses the md5
> protocol (this name better reflects the streaming property of the test
> as well), and change the matroska tests to use that, if you really want
> to test the streaming output.

Yes, that also works. The idea is to keep a working fate function that
can easily let us test muxers when using non seekable output.

> 
> BTW in the matroska muxer, should a non-seekable output and using the
> -live option produce the same result? Because there are some places in
> the code which only checks if -live is set, and I am not sure if that is
> intentional.

Probably not. All the code related to the -live option is nowadays
somwhat precarious after further development and merges took place in
the muxer.

> 
> Regards,
> Marton
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel@ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

Patch hide | download patch | download mbox

diff --git a/tests/fate-run.sh b/tests/fate-run.sh
index 931150b5a6..20dae9ea13 100755
--- a/tests/fate-run.sh
+++ b/tests/fate-run.sh
@@ -143,7 +143,10 @@  crc(){
 }
 
 md5(){
-    ffmpeg "$@" md5:
+    encfile="${outdir}/${test}.out"
+    cleanfiles=$encfile
+    ffmpeg "$@" $encfile
+    do_md5sum $encfile | awk '{print $1}'
 }
 
 pcm(){
diff --git a/tests/fate/filter-audio.mak b/tests/fate/filter-audio.mak
index 5d15b31e0b..8030641ed6 100644
--- a/tests/fate/filter-audio.mak
+++ b/tests/fate/filter-audio.mak
@@ -215,7 +215,7 @@  fate-filter-channelmap-one-int: SRC = $(TARGET_PATH)/tests/data/asynth-44100-6.w
 fate-filter-channelmap-one-int: tests/data/asynth-44100-6.wav
 fate-filter-channelmap-one-int: CMD = md5 -i $(SRC) -filter_complex_script $(TARGET_PATH)/tests/data/filtergraphs/channelmap_one_int -f wav -fflags +bitexact
 fate-filter-channelmap-one-int: CMP = oneline
-fate-filter-channelmap-one-int: REF = 428b8f9fac6d57147069b97335019ef5
+fate-filter-channelmap-one-int: REF = 8cfe553d65ed4696756d8c1b824fcdd3
 
 FATE_FILTER_CHANNELMAP += fate-filter-channelmap-one-str
 fate-filter-channelmap-one-str: tests/data/filtergraphs/channelmap_one_str
@@ -223,7 +223,7 @@  fate-filter-channelmap-one-str: SRC = $(TARGET_PATH)/tests/data/asynth-44100-2.w
 fate-filter-channelmap-one-str: tests/data/asynth-44100-2.wav
 fate-filter-channelmap-one-str: CMD = md5 -i $(SRC) -filter_complex_script $(TARGET_PATH)/tests/data/filtergraphs/channelmap_one_str -f wav -fflags +bitexact
 fate-filter-channelmap-one-str: CMP = oneline
-fate-filter-channelmap-one-str: REF = e788890db6a11c2fb29d7c4229072d49
+fate-filter-channelmap-one-str: REF = 0ea3052e482c95d5d3bd9da6dac1b5fa
 
 FATE_AFILTER-$(call FILTERDEMDECENCMUX, CHANNELMAP, WAV, PCM_S16LE, PCM_S16LE, WAV) += $(FATE_FILTER_CHANNELMAP)
 
diff --git a/tests/fate/matroska.mak b/tests/fate/matroska.mak
index 48b710c26c..9e233cf510 100644
--- a/tests/fate/matroska.mak
+++ b/tests/fate/matroska.mak
@@ -4,7 +4,7 @@ 
 FATE_MATROSKA-$(call DEMMUX, MATROSKA, MATROSKA) += fate-matroska-remux
 fate-matroska-remux: CMD = md5 -i $(TARGET_SAMPLES)/vp9-test-vectors/vp90-2-2pass-akiyo.webm -color_trc 4 -c:v copy -fflags +bitexact -strict -2 -f matroska
 fate-matroska-remux: CMP = oneline
-fate-matroska-remux: REF = 1ed49a4f2b6790357fac268938357353
+fate-matroska-remux: REF = 53be067cfaee3626d1c94b4a0fd5ac10
 
 FATE_MATROSKA_FFPROBE-$(call ALLYES, MATROSKA_DEMUXER) += fate-matroska-spherical-mono
 fate-matroska-spherical-mono: CMD = run ffprobe$(PROGSSUF)$(EXESUF) -show_entries stream_side_data_list -select_streams v -v 0 $(TARGET_SAMPLES)/mkv/spherical.mkv
diff --git a/tests/fate/wavpack.mak b/tests/fate/wavpack.mak
index 32ae3f6100..1cb414447c 100644
--- a/tests/fate/wavpack.mak
+++ b/tests/fate/wavpack.mak
@@ -91,12 +91,12 @@  fate-wavpack-matroskamode: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/special/matros
 FATE_WAVPACK-$(call DEMMUX, WV, MATROSKA) += fate-wavpack-matroska_mux-mono
 fate-wavpack-matroska_mux-mono: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/num_channels/mono_16bit_int.wv -c copy -fflags +bitexact -f matroska
 fate-wavpack-matroska_mux-mono: CMP = oneline
-fate-wavpack-matroska_mux-mono: REF = 11773e2a518edc788475f3880d849230
+fate-wavpack-matroska_mux-mono: REF = 31c0b4e71b532103bbfd2e284795cac1
 
 FATE_WAVPACK-$(call DEMMUX, WV, MATROSKA) += fate-wavpack-matroska_mux-61
 fate-wavpack-matroska_mux-61: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/num_channels/eva_2.22_6.1_16bit-partial.wv -c copy -fflags +bitexact -f matroska
 fate-wavpack-matroska_mux-61: CMP = oneline
-fate-wavpack-matroska_mux-61: REF = 9641abdf596c10c2e21bd9b026d4bade
+fate-wavpack-matroska_mux-61: REF = a0e5ad857d57a50869c21f1621e55674
 
 FATE_SAMPLES_AVCONV += $(FATE_WAVPACK-yes)
 fate-wavpack: $(FATE_WAVPACK-yes)
diff --git a/tests/ref/fate/binsub-mksenc b/tests/ref/fate/binsub-mksenc
index f247d9d22d..a0a58b880d 100644
--- a/tests/ref/fate/binsub-mksenc
+++ b/tests/ref/fate/binsub-mksenc
@@ -1 +1 @@ 
-f80f42e646fce972e73aa6d99dcfa470
+cdc96049a5aeaccec0126f19600d88aa
diff --git a/tests/ref/fate/mapchan-6ch-extract-2-downmix-mono b/tests/ref/fate/mapchan-6ch-extract-2-downmix-mono
index a47c85cd6b..9ac4dba261 100644
--- a/tests/ref/fate/mapchan-6ch-extract-2-downmix-mono
+++ b/tests/ref/fate/mapchan-6ch-extract-2-downmix-mono
@@ -1 +1 @@ 
-2727e5418eb5b8f89954f140d9f3a19a
+6ededcfc9f0b26d200a7295205e87663
diff --git a/tests/ref/fate/mapchan-silent-mono b/tests/ref/fate/mapchan-silent-mono
index f720c318f4..93f519e0ee 100644
--- a/tests/ref/fate/mapchan-silent-mono
+++ b/tests/ref/fate/mapchan-silent-mono
@@ -1 +1 @@ 
-16dc73c037897dff9b1d3e3d8561dc73
+b6b034fffa8639e64d79be84f16d9ebe
diff --git a/tests/ref/fate/mpeg4-bsf-unpack-bframes b/tests/ref/fate/mpeg4-bsf-unpack-bframes
index 162d436625..703ae3a8e8 100644
--- a/tests/ref/fate/mpeg4-bsf-unpack-bframes
+++ b/tests/ref/fate/mpeg4-bsf-unpack-bframes
@@ -1 +1 @@ 
-c9535e459c2ee4ead6d84b93bc7e9f46
+a0c4ba21ab9d68966b8fce7cc848e2c6
diff --git a/tests/ref/fate/time_base b/tests/ref/fate/time_base
index eec4b21aa5..7923556b35 100644
--- a/tests/ref/fate/time_base
+++ b/tests/ref/fate/time_base
@@ -1 +1 @@ 
-ef9896ce045941607782e895aea930b6
+d26a35b141551b36c5b8bd716451cfcb
diff --git a/tests/ref/fate/v410enc b/tests/ref/fate/v410enc
index 139da7b875..9fddf5a35c 100644
--- a/tests/ref/fate/v410enc
+++ b/tests/ref/fate/v410enc
@@ -1 +1 @@ 
-5fd2d9a7b3311f5c19dbdd647bb9eae6
+465bcc7477104a8295f47b35f1b987df