[FFmpeg-devel] [PATCH v2] fate: Allow running multiple rounds of tests with differing settings

[Martin Storsjö]

This can be used to run tests multple times, with e.g. differing
QEMU settings, by adding something like this to the FATE configuration
file:

    target_exec="qemu-aarch64-static"
    fate_targets="fate-checkasm fate-cpu"

    fate_environments="sve128 sve256 sve512"
    sve128_env="QEMU_CPU=max,sve128=on"
    sve256_env="QEMU_CPU=max,sve256=on"
    sve512_env="QEMU_CPU=max,sve512=on"

It's also possible to customize the target_exec command further
by injecting a sufficiently quoted variable into it, which then can
be updated for each run, e.g. target_exec="\$(CUR_EXEC_CMD)".

For each of the environment names in fate_environments, the tests
that are run get the name suffixed on the fate tests in the
test log and fate report, e.g. "fate-checkasm-h264dsp_sve128".
---
v2: Added examples to fate_config.sh.template.

Will push this together with
https://patchwork.ffmpeg.org/project/ffmpeg/patch/20231127123118.3622784-1-martin@martin.st/
soon if this seems reasonable.
---
 doc/fate_config.sh.template | 20 ++++++++++++++++++++
 tests/Makefile              |  4 ++--
 tests/fate.sh               | 14 +++++++++++++-
 3 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/doc/fate_config.sh.template b/doc/fate_config.sh.template
index 06bb79a832..8681805d65 100644
--- a/doc/fate_config.sh.template
+++ b/doc/fate_config.sh.template
@@ -33,3 +33,23 @@ makeopts=       # extra options passed to 'make'
                 # defaults to 'tar c'
 #fate_targets=  # targets to make when running fate; defaults to "fate",
                 # can be set to run a subset of tests, e.g. "fate-checkasm".
+
+#fate_environments=  # a list of names of configurations to run tests for;
+                     # each round is run with variables from ${${name}_env} set.
+
+# One example of using fate_environments:
+
+# target_exec="qemu-aarch64-static"
+# fate_targets="fate-checkasm fate-cpu"
+# fate_environments="sve128 sve256"
+# sve128_env="QEMU_CPU=max,sve128=on"
+# sve256_env="QEMU_CPU=max,sve256=on"
+
+# The variables set by fate_environments can also be used explicitly
+# by target_exec, e.g. like this:
+
+# target_exec="qemu-aarch64-static -cpu \$(MY_CPU)"
+# fate_targets="fate-checkasm fate-cpu"
+# fate_environments="sve128 sve256"
+# sve128_env="MY_CPU=max,sve128=on"
+# sve256_env="MY_CPU=max,sve256=on"
diff --git a/tests/Makefile b/tests/Makefile
index 444c09b3de..744dbcdfb3 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -305,8 +305,8 @@ $(FATE): export PROGSUF = $(PROGSSUF)
 $(FATE): export EXECSUF = $(EXESUF)
 $(FATE): export HOSTEXECSUF = $(HOSTEXESUF)
 $(FATE): $(FATE_UTILS:%=tests/%$(HOSTEXESUF)) | $(FATE_OUTDIRS)
-	@echo "TEST    $(@:fate-%=%)"
-	$(Q)$(SRC_PATH)/tests/fate-run.sh $@ "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
+	@echo "TEST    $(@:fate-%=%)$(FATE_SUFFIX)"
+	$(Q)$(SRC_PATH)/tests/fate-run.sh $@$(FATE_SUFFIX) "$(TARGET_SAMPLES)" "$(TARGET_EXEC)" "$(TARGET_PATH)" '$(CMD)' '$(CMP)' '$(REF)' '$(FUZZ)' '$(THREADS)' '$(THREAD_TYPE)' '$(CPUFLAGS)' '$(CMP_SHIFT)' '$(CMP_TARGET)' '$(SIZE_TOLERANCE)' '$(CMP_UNIT)' '$(GEN)' '$(HWACCEL)' '$(REPORT)' '$(KEEP_FILES)'
 
 fate-list:
 	@printf '%s\n' $(sort $(FATE))
diff --git a/tests/fate.sh b/tests/fate.sh
index 95408ea109..c04eb41cbe 100755
--- a/tests/fate.sh
+++ b/tests/fate.sh
@@ -101,7 +101,19 @@ compile_extra()(
 fate()(
     test "$build_only" = "yes" && return
     cd ${build} || return
-    ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
+    if [ -n "${fate_environments}" ]; then
+        ret=0
+        for e in ${fate_environments}; do
+            eval "curenv=\${${e}_env}"
+            echo Testing environment ${e}: ${curenv}
+            ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets} FATE_SUFFIX=_${e} ${curenv}
+            cur_ret=$?
+            test $cur_ret != 0 && ret=$cur_ret
+        done
+        return $ret
+    else
+        ${make} ${makeopts_fate-${makeopts}} -k ${fate_targets}
+    fi
 )
 
 clean(){
-- 
2.34.1