mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

gitlab: add optional job to run flaky avocado tests

Browse Source 
One problem with flaky tests is they often only fail under CI
conditions which makes it hard to debug. We add an optional allow_fail
job so developers can trigger the only the flaky tests in the CI
environment if they are debugging.

Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Acked-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
Message-Id: <20231201093633.2551497-8-alex.bennee@linaro.org>
This commit is contained in:
Alex Bennée 2023-12-01 09:36:33 +00:00
parent aeb5f8f248
commit 5d25fcb702
14 changed files with 57 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.gitlab-ci.d/buildtest.yml
View File

@ -217,6 +217,36 @@ avocado-system-opensuse:
MAKE_CHECK_ARGS: check-avocado
AVOCADO_TAGS: arch:s390x arch:x86_64 arch:aarch64
#
# Flaky tests. We don't run these by default and they are allow fail
# but often the CI system is the only way to trigger the failures.
#
build-system-flaky:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
job: amd64-debian-container
variables:
IMAGE: debian
QEMU_JOB_OPTIONAL: 1
TARGETS: aarch64-softmmu arm-softmmu mips64el-softmmu
ppc64-softmmu rx-softmmu s390x-softmmu sh4-softmmu x86_64-softmmu
MAKE_CHECK_ARGS: check-build
avocado-system-flaky:
extends: .avocado_test_job_template
needs:
- job: build-system-flaky
artifacts: true
allow_failure: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check-avocado
QEMU_JOB_OPTIONAL: 1
QEMU_TEST_FLAKY_TESTS: 1
AVOCADO_TAGS: flaky
# This jobs explicitly disable TCG (--disable-tcg), KVM is detected by
# the configure script. The container doesn't contain Xen headers so

15
docs/devel/testing.rst
View File

@ -1387,16 +1387,17 @@ the code snippet below:
def test(self):
do_something()
Tests should not live in this state forever and should either be fixed
or eventually removed.
To run such tests locally you will need to set the environment
variable. For example:
You can also add ``:avocado: tags=flaky`` to the test meta-data so
only the flaky tests can be run as a group:
.. code::
env QEMU_TEST_FLAKY_TESTS=1 ./pyvenv/bin/avocado run \
tests/avocado/boot_linux.py:BootLinuxPPC64.test_pseries_tcg
env QEMU_TEST_FLAKY_TESTS=1 ./pyvenv/bin/avocado \
run tests/avocado -filter-by-tags=flaky
Tests should not live in this state forever and should either be fixed
or eventually removed.
Uninstalling Avocado
~~~~~~~~~~~~~~~~~~~~

2
tests/avocado/boot_linux.py
View File

@ -99,6 +99,7 @@ class BootLinuxPPC64(LinuxTest):
"""
:avocado: tags=machine:pseries
:avocado: tags=accel:tcg
:avocado: tags=flaky
"""
self.require_accelerator("tcg")
self.vm.add_args("-accel", "tcg")
@ -118,6 +119,7 @@ class BootLinuxS390X(LinuxTest):
"""
:avocado: tags=machine:s390-ccw-virtio
:avocado: tags=accel:tcg
:avocado: tags=flaky
"""
self.require_accelerator("tcg")
self.vm.add_args("-accel", "tcg")

1
tests/avocado/boot_linux_console.py
View File

@ -1425,6 +1425,7 @@ class BootLinuxConsole(LinuxKernelTest):
"""
:avocado: tags=arch:sh4
:avocado: tags=machine:r2d
:avocado: tags=flaky
"""
tar_hash = 'fe06a4fd8ccbf2e27928d64472939d47829d4c7e'
self.vm.add_args('-append', 'console=ttySC1')

1
tests/avocado/intel_iommu.py
View File

@ -22,6 +22,7 @@ class IntelIOMMU(LinuxTest):
:avocado: tags=machine:q35
:avocado: tags=accel:kvm
:avocado: tags=intel_iommu
:avocado: tags=flaky
"""
IOMMU_ADDON = ',iommu_platform=on,disable-modern=off,disable-legacy=on'

2
tests/avocado/linux_initrd.py
View File

@ -57,6 +57,8 @@ class LinuxInitrd(QemuSystemTest):
def test_with_2gib_file_should_work_with_linux_v4_16(self):
"""
:avocado: tags=flaky
QEMU has supported up to 4 GiB initrd for recent kernel
Expect guest can reach 'Unpacking initramfs...'
"""

2
tests/avocado/machine_aspeed.py
View File

@ -317,6 +317,7 @@ class AST2x00MachineSDK(QemuSystemTest, LinuxSSHMixIn):
"""
:avocado: tags=arch:arm
:avocado: tags=machine:ast2500-evb
:avocado: tags=flaky
"""
image_url = ('https://github.com/AspeedTech-BMC/openbmc/releases/'
@ -336,6 +337,7 @@ class AST2x00MachineSDK(QemuSystemTest, LinuxSSHMixIn):
"""
:avocado: tags=arch:arm
:avocado: tags=machine:ast2600-evb
:avocado: tags=flaky
"""
image_url = ('https://github.com/AspeedTech-BMC/openbmc/releases/'

2
tests/avocado/machine_mips_malta.py
View File

@ -109,6 +109,7 @@ class MaltaMachineFramebuffer(QemuSystemTest):
:avocado: tags=machine:malta
:avocado: tags=cpu:I6400
:avocado: tags=mips:smp
:avocado: tags=flaky
"""
self.do_test_i6400_framebuffer_logo(7)
@ -120,6 +121,7 @@ class MaltaMachineFramebuffer(QemuSystemTest):
:avocado: tags=machine:malta
:avocado: tags=cpu:I6400
:avocado: tags=mips:smp
:avocado: tags=flaky
"""
self.do_test_i6400_framebuffer_logo(8)

2
tests/avocado/machine_rx_gdbsim.py
View File

@ -31,6 +31,7 @@ class RxGdbSimMachine(QemuSystemTest):
:avocado: tags=arch:rx
:avocado: tags=machine:gdbsim-r5f562n8
:avocado: tags=endian:little
:avocado: tags=flaky
"""
uboot_url = ('https://acc.dl.osdn.jp/users/23/23888/u-boot.bin.gz')
uboot_hash = '9b78dbd43b40b2526848c0b1ce9de02c24f4dcdb'
@ -56,6 +57,7 @@ class RxGdbSimMachine(QemuSystemTest):
:avocado: tags=arch:rx
:avocado: tags=machine:gdbsim-r5f562n7
:avocado: tags=endian:little
:avocado: tags=flaky
"""
dtb_url = ('https://acc.dl.osdn.jp/users/23/23887/rx-virt.dtb')
dtb_hash = '7b4e4e2c71905da44e86ce47adee2210b026ac18'

1
tests/avocado/machine_s390_ccw_virtio.py
View File

@ -167,6 +167,7 @@ class S390CCWVirtioMachine(QemuSystemTest):
:avocado: tags=device:virtio-gpu
:avocado: tags=device:virtio-crypto
:avocado: tags=device:virtio-net
:avocado: tags=flaky
"""
kernel_url = ('https://archives.fedoraproject.org/pub/archive'

2
tests/avocado/replay_kernel.py
View File

@ -88,6 +88,7 @@ class ReplayKernelNormal(ReplayKernelBase):
"""
:avocado: tags=arch:x86_64
:avocado: tags=machine:pc
:avocado: tags=flaky
"""
kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
'/linux/releases/29/Everything/x86_64/os/images/pxeboot'
@ -186,6 +187,7 @@ class ReplayKernelNormal(ReplayKernelBase):
"""
:avocado: tags=arch:arm
:avocado: tags=machine:cubieboard
:avocado: tags=flaky
"""
deb_url = ('https://apt.armbian.com/pool/main/l/'
'linux-5.10.16-sunxi/linux-image-current-sunxi_21.02.2_armhf.deb')

2
tests/avocado/reverse_debugging.py
View File

@ -255,6 +255,7 @@ class ReverseDebugging_ppc64(ReverseDebugging):
"""
:avocado: tags=arch:ppc64
:avocado: tags=machine:pseries
:avocado: tags=flaky
"""
# SLOF branches back to its entry point, which causes this test
# to take the 'hit a breakpoint again' path. That's not a problem,
@ -269,6 +270,7 @@ class ReverseDebugging_ppc64(ReverseDebugging):
"""
:avocado: tags=arch:ppc64
:avocado: tags=machine:powernv
:avocado: tags=flaky
"""
self.endian_is_le = False
self.reverse_debugging()

1
tests/avocado/smmu.py
View File

@ -22,6 +22,7 @@ class SMMU(LinuxTest):
:avocado: tags=machine:virt
:avocado: tags=distro:fedora
:avocado: tags=smmu
:avocado: tags=flaky
"""
IOMMU_ADDON = ',iommu_platform=on,disable-modern=off,disable-legacy=on'

1
tests/avocado/tuxrun_baselines.py
View File

@ -561,6 +561,7 @@ class TuxRunBaselineTest(QemuSystemTest):
:avocado: tags=image:zImage
:avocado: tags=root:sda
:avocado: tags=console:ttySC1
:avocado: tags=flaky
"""
sums = { "rootfs.ext4.zst" :
"3592a7a3d5a641e8b9821449e77bc43c9904a56c30d45da0694349cfd86743fd",