@@ -33,3 +33,23 @@ makeopts= # extra options passed to 'make'
# defaults to 'tar c'
#fate_targets= # targets to make when running fate; defaults to "fate",
# can be set to run a subset of tests, e.g. "fate-checkasm".
+
+#fate_environments= # a list of names of configurations to run tests for;
+ # each round is run with variables from ${${name}_env} set.
+
+# One example of using fate_environments:
+
+# target_exec="qemu-aarch64-static"
+# fate_targets="fate-checkasm fate-cpu"
+# fate_environments="sve128 sve256"
+# sve128_env="QEMU_CPU=max,sve128=on"
+# sve256_env="QEMU_CPU=max,sve256=on"
+
+# The variables set by fate_environments can also be used explicitly
+# by target_exec, e.g. like this:
+
+# target_exec="qemu-aarch64-static -cpu \$(MY_CPU)"
+# fate_targets="fate-checkasm fate-cpu"
+# fate_environments="sve128 sve256"
+# sve128_env="MY_CPU=max,sve128=on"
+# sve256_env="MY_CPU=max,sve256=on"
@@ -305,8 +305,8 @@ $(FATE): export PROGSUF = $(PROGSSUF)
$(FATE): export EXECSUF = $(EXESUF)
$(FATE): export HOSTEXECSUF = $(HOSTEXESUF)
$(FATE): $(FATE_UTILS:%=tests/%$(HOSTEXESUF)) | $(FATE_OUTDIRS)
- @echo "TEST $(@:fate-%=%)"
- $(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
+ @echo "TEST $(@:fate-%=%)$(FATE_SUFFIX)"
+ $(Q)$(SRC_PATH)/tests/fate-run.sh $@$(FATE_SUFFIX) "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
fate-list:
@printf '%s\n' $(sort $(FATE))
@@ -101,7 +101,19 @@ compile_extra()(
fate()(
test "$build_only" = "yes" && return
cd ${build} || return
- ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
+ if [ -n "${fate_environments}" ]; then
+ ret=0
+ for e in ${fate_environments}; do
+ eval "curenv=\${${e}_env}"
+ echo Testing environment ${e}: ${curenv}
+ ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets} FATE_SUFFIX=_${e} ${curenv}
+ cur_ret=$?
+ test $cur_ret != 0 && ret=$cur_ret
+ done
+ return $ret
+ else
+ ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
+ fi
)
clean(){
This can be used to run tests multple times, with e.g. differing QEMU settings, by adding something like this to the FATE configuration file: target_exec="qemu-aarch64-static" fate_targets="fate-checkasm fate-cpu" fate_environments="sve128 sve256 sve512" sve128_env="QEMU_CPU=max,sve128=on" sve256_env="QEMU_CPU=max,sve256=on" sve512_env="QEMU_CPU=max,sve512=on" It's also possible to customize the target_exec command further by injecting a sufficiently quoted variable into it, which then can be updated for each run, e.g. target_exec="\$(CUR_EXEC_CMD)". For each of the environment names in fate_environments, the tests that are run get the name suffixed on the fate tests in the test log and fate report, e.g. "fate-checkasm-h264dsp_sve128". --- v2: Added examples to fate_config.sh.template. Will push this together with https://patchwork.ffmpeg.org/project/ffmpeg/patch/20231127123118.3622784-1-martin@martin.st/ soon if this seems reasonable. --- doc/fate_config.sh.template | 20 ++++++++++++++++++++ tests/Makefile | 4 ++-- tests/fate.sh | 14 +++++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-)