mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
21b1ee7691
qemu/tests/Makefile.include
Daniel P. Berrangé f57213f85b tests/functional: enable pre-emptive caching of assets 
Many tests need to access assets stored on remote sites. We don't want
to download these during test execution when run by meson, since this
risks hitting test timeouts when data transfers are slow.

Add support for pre-emptive caching of assets by setting the env var
QEMU_TEST_PRECACHE to point to a timestamp file. When this is set,
instead of running the test, the assets will be downloaded and saved
to the cache, then the timestamp file created.

A meson custom target is created as a dependency of each test suite
to trigger the pre-emptive caching logic before the test runs.

When run in caching mode, it will locate assets by looking for class
level variables with a name prefix "ASSET_", and type "Asset".

At the ninja level

   ninja test --suite functional

will speculatively download any assets that are not already cached,
so it is advisable to set a timeout multiplier.

   QEMU_TEST_NO_DOWNLOAD=1 ninja test --suite functional

will fail the test if a required asset is not already cached

   ninja precache-functional

will download and cache all assets required by the functional
tests

At the make level, precaching is always done by

   make check-functional

Signed-off-by: Daniel P. Berrangé <berrange@redhat.com>
Tested-by: Richard Henderson <richard.henderson@linaro.org>
[thuth: Remove the duplicated "path = os.path.basename(...)" line]
Message-ID: <20240830133841.142644-16-thuth@redhat.com>
Signed-off-by: Thomas Huth <thuth@redhat.com>
2024-09-04 10:52:29 +02:00

181 lines
6.6 KiB
Makefile
Raw Blame History

# -*- Mode: makefile -*-
.PHONY: check-help
check-help:
@echo "Regression testing targets:"
@echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests"
@echo " $(MAKE) bench Run speed tests"
@echo
@echo "Individual test suites:"
@echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target"
@echo " $(MAKE) check-qtest Run qtest tests"
@echo " $(MAKE) check-functional Run python-based functional tests"
@echo " $(MAKE) check-functional-TARGET Run functional tests for a given target"
@echo " $(MAKE) check-unit Run qobject tests"
@echo " $(MAKE) check-qapi-schema Run QAPI schema tests"
@echo " $(MAKE) check-block Run block tests"
ifneq ($(filter $(all-check-targets), check-softfloat),)
@echo " $(MAKE) check-tcg Run TCG tests"
@echo " $(MAKE) check-softfloat Run FPU emulation tests"
endif
@echo " $(MAKE) check-avocado Run avocado (integration) tests for currently configured targets"
@echo
@echo " $(MAKE) check-report.junit.xml Generates an aggregated XML test report"
@echo " $(MAKE) check-venv Creates a Python venv for tests"
@echo " $(MAKE) check-clean Clean the tests and related data"
@echo
@echo "The following are useful for CI builds"
@echo " $(MAKE) check-build Build most test binaries"
@echo " $(MAKE) get-vm-images Downloads all images used by avocado tests, according to configured targets (~350 MB each, 1.5 GB max)"
@echo
@echo
@echo "The variable SPEED can be set to control the gtester speed setting."
@echo "Default options are -k and (for $(MAKE) V=1) --verbose; they can be"
@echo "changed with variable GTESTER_OPTIONS."
ifneq ($(wildcard config-host.mak),)
export SRC_PATH
SPEED = quick
# Per guest TCG tests
BUILD_TCG_TARGET_RULES=$(patsubst %,build-tcg-tests-%, $(TCG_TESTS_TARGETS))
CLEAN_TCG_TARGET_RULES=$(patsubst %,clean-tcg-tests-%, $(TCG_TESTS_TARGETS))
DISTCLEAN_TCG_TARGET_RULES=$(patsubst %,distclean-tcg-tests-%, $(TCG_TESTS_TARGETS))
RUN_TCG_TARGET_RULES=$(patsubst %,run-tcg-tests-%, $(TCG_TESTS_TARGETS))
$(foreach TARGET,$(TCG_TESTS_TARGETS), \
$(eval $(BUILD_DIR)/tests/tcg/config-$(TARGET).mak: config-host.mak))
.PHONY: $(TCG_TESTS_TARGETS:%=build-tcg-tests-%)
$(TCG_TESTS_TARGETS:%=build-tcg-tests-%): build-tcg-tests-%: $(BUILD_DIR)/tests/tcg/config-%.mak
$(call quiet-command, \
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS), \
"BUILD","$* guest-tests")
.PHONY: $(TCG_TESTS_TARGETS:%=run-tcg-tests-%)
$(TCG_TESTS_TARGETS:%=run-tcg-tests-%): run-tcg-tests-%: build-tcg-tests-%
$(call quiet-command, \
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) SPEED=$(SPEED) run, \
"RUN", "$* guest-tests")
.PHONY: $(TCG_TESTS_TARGETS:%=clean-tcg-tests-%)
$(TCG_TESTS_TARGETS:%=clean-tcg-tests-%): clean-tcg-tests-%:
$(call quiet-command, \
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) clean, \
"CLEAN", "$* guest-tests")
.PHONY: $(TCG_TESTS_TARGETS:%=distclean-tcg-tests-%)
$(TCG_TESTS_TARGETS:%=distclean-tcg-tests-%): distclean-tcg-tests-%:
$(call quiet-command, \
$(MAKE) -C tests/tcg/$* $(SUBDIR_MAKEFLAGS) distclean, \
"CLEAN", "$* guest-tests")
.PHONY: build-tcg
build-tcg: $(BUILD_TCG_TARGET_RULES)
.PHONY: check-tcg
.ninja-goals.check-tcg = all test-plugins
check-tcg: $(RUN_TCG_TARGET_RULES)
.PHONY: clean-tcg
clean-tcg: $(CLEAN_TCG_TARGET_RULES)
.PHONY: distclean-tcg
distclean-tcg: $(DISTCLEAN_TCG_TARGET_RULES)
# Python venv for running tests
.PHONY: check-venv check-avocado check-acceptance check-acceptance-deprecated-warning
# Build up our target list from the filtered list of ninja targets
TARGETS=$(patsubst libqemu-%.a, %, $(filter libqemu-%.a, $(ninja-targets)))
TESTS_VENV_TOKEN=$(BUILD_DIR)/pyvenv/tests.group
TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results
ifndef AVOCADO_TESTS
AVOCADO_TESTS=tests/avocado
endif
# Controls the output generated by Avocado when running tests.
# Any number of command separated loggers are accepted. For more
# information please refer to "avocado --help".
AVOCADO_SHOW?=app
ifndef AVOCADO_TAGS
AVOCADO_CMDLINE_TAGS=$(patsubst %-softmmu,-t arch:%, \
$(filter %-softmmu,$(TARGETS)))
else
AVOCADO_CMDLINE_TAGS=$(addprefix -t , $(AVOCADO_TAGS))
endif
quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \
$(PYTHON) -m pip -q --disable-pip-version-check $1, \
"VENVPIP","$1")
$(TESTS_VENV_TOKEN): $(SRC_PATH)/pythondeps.toml
$(call quiet-venv-pip,install -e "$(SRC_PATH)/python/")
$(MKVENV_ENSUREGROUP) $< avocado
$(call quiet-command, touch $@)
$(TESTS_RESULTS_DIR):
$(call quiet-command, mkdir -p $@, \
MKDIR, $@)
check-venv: $(TESTS_VENV_TOKEN)
FEDORA_31_ARCHES_TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGETS)))
FEDORA_31_ARCHES_CANDIDATES=$(patsubst ppc64,ppc64le,$(FEDORA_31_ARCHES_TARGETS))
FEDORA_31_ARCHES := x86_64 aarch64 ppc64le s390x
FEDORA_31_DOWNLOAD=$(filter $(FEDORA_31_ARCHES),$(FEDORA_31_ARCHES_CANDIDATES))
# download one specific Fedora 31 image
get-vm-image-fedora-31-%: check-venv
$(call quiet-command, \
$(PYTHON) -m avocado vmimage get \
--distro=fedora --distro-version=31 --arch=$*, \
"AVOCADO", "Downloading avocado tests VM image for $*")
# download all vm images, according to defined targets
get-vm-images: check-venv $(patsubst %,get-vm-image-fedora-31-%, $(FEDORA_31_DOWNLOAD))
check-avocado: check-venv $(TESTS_RESULTS_DIR) get-vm-images
$(call quiet-command, \
$(PYTHON) -m avocado \
--show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \
$(if $(AVOCADO_TAGS),, --filter-by-tags-include-empty \
--filter-by-tags-include-empty-key) \
$(AVOCADO_CMDLINE_TAGS) --max-parallel-tasks=1 \
$(if $(GITLAB_CI),,--failfast) $(AVOCADO_TESTS), \
"AVOCADO", "tests/avocado")
check-acceptance-deprecated-warning:
@echo
@echo "Note '$(MAKE) check-acceptance' is deprecated, use '$(MAKE) check-avocado' instead."
@echo
check-acceptance: check-acceptance-deprecated-warning | check-avocado
FUNCTIONAL_TARGETS=$(patsubst %-softmmu,check-functional-%, $(filter %-softmmu,$(TARGETS)))
.PHONY: $(FUNCTIONAL_TARGETS)
$(FUNCTIONAL_TARGETS):
@$(MAKE) SPEED=thorough $(subst -functional,-func,$@)
.PHONY: check-functional
check-functional:
@$(NINJA) precache-functional
@QEMU_TEST_NO_DOWNLOAD=1 $(MAKE) SPEED=thorough check-func check-func-quick
# Consolidated targets
.PHONY: check check-clean get-vm-images
check:
check-build: run-ninja
check-clean:
rm -rf $(TESTS_RESULTS_DIR)
clean: check-clean clean-tcg
distclean: distclean-tcg
endif
Reference in New Issue View Git Blame Copy Permalink