[FFmpeg-devel,PATCHv3] fate: Add an option for disabling the 2k/4k tests

When testing on a memory limited system, these tests consume a
significant amount of memory and can often fail if testing by running
multiple processes in parallel.
---
Made it possible to override the setting when running make, by setting
LARGE_TESTS. The solution looks a bit different from what I mentioned
before as it didn't really work just like I guessed.
---
 configure             | 16 ++++++++++++++++
 tests/fate/seek.mak   |  3 ++-
 tests/fate/vcodec.mak |  5 +++--
 3 files changed, 21 insertions(+), 3 deletions(-)

On 12/15/2019 6:50 PM, Martin Storsjö wrote:
> When testing on a memory limited system, these tests consume a
> significant amount of memory and can often fail if testing by running
> multiple processes in parallel.
> ---
> Made it possible to override the setting when running make, by setting
> LARGE_TESTS. The solution looks a bit different from what I mentioned
> before as it didn't really work just like I guessed.
> ---
>  configure             | 16 ++++++++++++++++
>  tests/fate/seek.mak   |  3 ++-
>  tests/fate/vcodec.mak |  5 +++--
>  3 files changed, 21 insertions(+), 3 deletions(-)
> 
> diff --git a/configure b/configure
> index 10c494e2a0..44f464c999 100755
> --- a/configure
> +++ b/configure
> @@ -482,6 +482,7 @@ Developer options (useful when working on FFmpeg itself):
>    --ignore-tests=TESTS     comma-separated list (without "fate-" prefix
>                             in the name) of tests whose result is ignored
>    --enable-linux-perf      enable Linux Performance Monitor API
> +  --disable-large-tests    disable tests that use a large amount of memory
>  
>  NOTE: Object files are built at the place where configure is launched.
>  EOF
> @@ -1931,6 +1932,7 @@ CONFIG_LIST="
>      $SUBSYSTEM_LIST
>      autodetect
>      fontconfig
> +    large_tests
>      linux_perf
>      memory_poisoning
>      neon_clobber_test
> @@ -3726,6 +3728,7 @@ enable asm
>  enable debug
>  enable doc
>  enable faan faandct faanidct
> +enable large_tests
>  enable optimizations
>  enable runtime_cpudetect
>  enable safe_bitstream_reader
> @@ -7492,6 +7495,19 @@ print_config CONFIG_ "$config_files" $CONFIG_LIST       \
>                                       $CONFIG_EXTRA      \
>                                       $ALL_COMPONENTS    \
>  
> +
> +cat >> ffbuild/config.mak <<EOF
> +# Allow overriding CONFIG_LARGE_TESTS via LARGE_TESTS, if set on the
> +# make command line.
> +ifeq (\$(LARGE_TESTS), yes)
> +CONFIG_LARGE_TESTS:=yes
> +!CONFIG_LARGE_TESTS:=
> +else ifeq (\$(LARGE_TESTS), no)
> +CONFIG_LARGE_TESTS:=
> +!CONFIG_LARGE_TESTS:=yes
> +endif

Why not add this to tests/Makefile instead of config.mak? It seems to
work after a quick test.

Should be ok either way.

> +EOF
> +
>  echo "#endif /* FFMPEG_CONFIG_H */" >> $TMPH
>  echo "endif # FFMPEG_CONFIG_MAK" >> ffbuild/config.mak
>  
> diff --git a/tests/fate/seek.mak b/tests/fate/seek.mak
> index 04f54ee6c4..98d2b54674 100644
> --- a/tests/fate/seek.mak
> +++ b/tests/fate/seek.mak
> @@ -64,7 +64,6 @@ FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, ASV1,          AVI)     += asv1
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, ASV2,          AVI)     += asv2
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, DNXHD,         DNXHD)   += dnxhd-720p
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, DNXHD,         DNXHD)   += dnxhd-720p-rd
> -FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, DNXHD,         DNXHD)   += dnxhd-4k-hr-lb
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, DNXHD,         MOV)     += dnxhd-1080i
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, DVVIDEO,       DV)      += dv
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, DVVIDEO,       DV)      += dv-411
> @@ -80,6 +79,8 @@ FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, JPEGLS,        AVI)     += jpegls
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, LJPEG MJPEG,   AVI)     += ljpeg
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, MJPEG,         AVI)     += mjpeg
>  
> +FATE_SEEK_VSYNTH_LENA-$(call ALLYES, DNXHD_ENCODER DNXHD_DECODER LARGE_TESTS) += dnxhd-4k-hr-lb
> +
>  FATE_SEEK_VSYNTH_LENA-$(call ENCDEC, MPEG1VIDEO, MPEG1VIDEO MPEGVIDEO) +=    \
>                                                      mpeg1                    \
>                                                      mpeg1b
> diff --git a/tests/fate/vcodec.mak b/tests/fate/vcodec.mak
> index 452246689e..fc27da5456 100644
> --- a/tests/fate/vcodec.mak
> +++ b/tests/fate/vcodec.mak
> @@ -29,13 +29,14 @@ FATE_VCODEC-$(call ENCDEC, DNXHD, DNXHD) += dnxhd-720p                  \
>                                              dnxhd-720p-rd               \
>                                              dnxhd-720p-10bit            \
>                                              dnxhd-720p-hr-lb            \
> -                                            dnxhd-4k-hr-lb              \
>                                              dnxhd-uhd-hr-sq             \
> -                                            dnxhd-2k-hr-hq              \
>                                              dnxhd-edge1-hr              \
>                                              dnxhd-edge2-hr              \
>                                              dnxhd-edge3-hr
>  
> +FATE_VCODEC-$(call ALLYES, DNXHD_ENCODER DNXHD_DECODER LARGE_TESTS) += dnxhd-4k-hr-lb \
> +                                                                       dnxhd-2k-hr-hq
> +
>  FATE_VCODEC-$(call ENCDEC, VC2 DIRAC, MOV) += vc2-420p vc2-420p10 vc2-420p12 \
>                                                vc2-422p vc2-422p10 vc2-422p12 \
>                                                vc2-444p vc2-444p10 vc2-444p12 \

On Mon, 16 Dec 2019, James Almer wrote:

> On 12/15/2019 6:50 PM, Martin Storsjö wrote:
>> When testing on a memory limited system, these tests consume a
>> significant amount of memory and can often fail if testing by running
>> multiple processes in parallel.
>> ---
>> Made it possible to override the setting when running make, by setting
>> LARGE_TESTS. The solution looks a bit different from what I mentioned
>> before as it didn't really work just like I guessed.
>> ---
>>  configure             | 16 ++++++++++++++++
>>  tests/fate/seek.mak   |  3 ++-
>>  tests/fate/vcodec.mak |  5 +++--
>>  3 files changed, 21 insertions(+), 3 deletions(-)
>> 
>> diff --git a/configure b/configure
>> index 10c494e2a0..44f464c999 100755
>> --- a/configure
>> +++ b/configure
>> @@ -482,6 +482,7 @@ Developer options (useful when working on FFmpeg itself):
>>    --ignore-tests=TESTS     comma-separated list (without "fate-" prefix
>>                             in the name) of tests whose result is ignored
>>    --enable-linux-perf      enable Linux Performance Monitor API
>> +  --disable-large-tests    disable tests that use a large amount of memory
>>
>>  NOTE: Object files are built at the place where configure is launched.
>>  EOF
>> @@ -1931,6 +1932,7 @@ CONFIG_LIST="
>>      $SUBSYSTEM_LIST
>>      autodetect
>>      fontconfig
>> +    large_tests
>>      linux_perf
>>      memory_poisoning
>>      neon_clobber_test
>> @@ -3726,6 +3728,7 @@ enable asm
>>  enable debug
>>  enable doc
>>  enable faan faandct faanidct
>> +enable large_tests
>>  enable optimizations
>>  enable runtime_cpudetect
>>  enable safe_bitstream_reader
>> @@ -7492,6 +7495,19 @@ print_config CONFIG_ "$config_files" $CONFIG_LIST       \
>>                                       $CONFIG_EXTRA      \
>>                                       $ALL_COMPONENTS    \
>> 
>> +
>> +cat >> ffbuild/config.mak <<EOF
>> +# Allow overriding CONFIG_LARGE_TESTS via LARGE_TESTS, if set on the
>> +# make command line.
>> +ifeq (\$(LARGE_TESTS), yes)
>> +CONFIG_LARGE_TESTS:=yes
>> +!CONFIG_LARGE_TESTS:=
>> +else ifeq (\$(LARGE_TESTS), no)
>> +CONFIG_LARGE_TESTS:=
>> +!CONFIG_LARGE_TESTS:=yes
>> +endif
>
> Why not add this to tests/Makefile instead of config.mak? It seems to
> work after a quick test.
>
> Should be ok either way.

Ah, yes, good point. Moved it there and pushed the patch. Thanks for the 
review!

// Martin