qemu/.gitlab-ci.d/buildtest.yml
Alex Bennée 5d25fcb702 gitlab: add optional job to run flaky avocado tests
One problem with flaky tests is they often only fail under CI
conditions which makes it hard to debug. We add an optional allow_fail
job so developers can trigger the only the flaky tests in the CI
environment if they are debugging.

Reviewed-by: Richard Henderson <richard.henderson@linaro.org>
Acked-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Alex Bennée <alex.bennee@linaro.org>
Message-Id: <20231201093633.2551497-8-alex.bennee@linaro.org>
2023-12-01 17:47:20 +00:00

662 lines
19 KiB
YAML

include:
- local: '/.gitlab-ci.d/buildtest-template.yml'
build-system-alpine:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
- job: amd64-alpine-container
variables:
IMAGE: alpine
TARGETS: avr-softmmu loongarch64-softmmu mips64-softmmu mipsel-softmmu
MAKE_CHECK_ARGS: check-build
CONFIGURE_ARGS: --enable-docs --enable-trace-backends=log,simple,syslog
check-system-alpine:
extends: .native_test_job_template
needs:
- job: build-system-alpine
artifacts: true
variables:
IMAGE: alpine
MAKE_CHECK_ARGS: check-unit check-qtest
avocado-system-alpine:
extends: .avocado_test_job_template
needs:
- job: build-system-alpine
artifacts: true
variables:
IMAGE: alpine
MAKE_CHECK_ARGS: check-avocado
AVOCADO_TAGS: arch:avr arch:loongarch64 arch:mips64 arch:mipsel
build-system-ubuntu:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
job: amd64-ubuntu2204-container
variables:
IMAGE: ubuntu2204
CONFIGURE_ARGS: --enable-docs
TARGETS: alpha-softmmu microblaze-softmmu mips64el-softmmu
MAKE_CHECK_ARGS: check-build
check-system-ubuntu:
extends: .native_test_job_template
needs:
- job: build-system-ubuntu
artifacts: true
variables:
IMAGE: ubuntu2204
MAKE_CHECK_ARGS: check
avocado-system-ubuntu:
extends: .avocado_test_job_template
needs:
- job: build-system-ubuntu
artifacts: true
variables:
IMAGE: ubuntu2204
MAKE_CHECK_ARGS: check-avocado
AVOCADO_TAGS: arch:alpha arch:microblaze arch:mips64el
build-system-debian:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
job: amd64-debian-container
variables:
IMAGE: debian
CONFIGURE_ARGS: --with-coroutine=sigaltstack
TARGETS: arm-softmmu i386-softmmu riscv64-softmmu sh4eb-softmmu
sparc-softmmu xtensa-softmmu
MAKE_CHECK_ARGS: check-build
check-system-debian:
extends: .native_test_job_template
needs:
- job: build-system-debian
artifacts: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check
avocado-system-debian:
extends: .avocado_test_job_template
needs:
- job: build-system-debian
artifacts: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check-avocado
AVOCADO_TAGS: arch:arm arch:i386 arch:riscv64 arch:sh4 arch:sparc arch:xtensa
crash-test-debian:
extends: .native_test_job_template
needs:
- job: build-system-debian
artifacts: true
variables:
IMAGE: debian
script:
- cd build
- make NINJA=":" check-venv
- pyvenv/bin/python3 scripts/device-crash-test -q --tcg-only ./qemu-system-i386
build-system-fedora:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
job: amd64-fedora-container
variables:
IMAGE: fedora
CONFIGURE_ARGS: --disable-gcrypt --enable-nettle --enable-docs
TARGETS: microblaze-softmmu mips-softmmu
xtensa-softmmu m68k-softmmu riscv32-softmmu ppc-softmmu sparc64-softmmu
MAKE_CHECK_ARGS: check-build
check-system-fedora:
extends: .native_test_job_template
needs:
- job: build-system-fedora
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check
avocado-system-fedora:
extends: .avocado_test_job_template
needs:
- job: build-system-fedora
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado
AVOCADO_TAGS: arch:microblaze arch:mips arch:xtensa arch:m68k
arch:riscv32 arch:ppc arch:sparc64
crash-test-fedora:
extends: .native_test_job_template
needs:
- job: build-system-fedora
artifacts: true
variables:
IMAGE: fedora
script:
- cd build
- make NINJA=":" check-venv
- pyvenv/bin/python3 scripts/device-crash-test -q ./qemu-system-ppc
- pyvenv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32
build-system-centos:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
job: amd64-centos8-container
variables:
IMAGE: centos8
CONFIGURE_ARGS: --disable-nettle --enable-gcrypt --enable-vfio-user-server
--enable-modules --enable-trace-backends=dtrace --enable-docs
TARGETS: ppc64-softmmu or1k-softmmu s390x-softmmu
x86_64-softmmu rx-softmmu sh4-softmmu nios2-softmmu
MAKE_CHECK_ARGS: check-build
check-system-centos:
extends: .native_test_job_template
needs:
- job: build-system-centos
artifacts: true
variables:
IMAGE: centos8
MAKE_CHECK_ARGS: check
avocado-system-centos:
extends: .avocado_test_job_template
needs:
- job: build-system-centos
artifacts: true
variables:
IMAGE: centos8
MAKE_CHECK_ARGS: check-avocado
AVOCADO_TAGS: arch:ppc64 arch:or1k arch:390x arch:x86_64 arch:rx
arch:sh4 arch:nios2
build-system-opensuse:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
job: amd64-opensuse-leap-container
variables:
IMAGE: opensuse-leap
TARGETS: s390x-softmmu x86_64-softmmu aarch64-softmmu
MAKE_CHECK_ARGS: check-build
check-system-opensuse:
extends: .native_test_job_template
needs:
- job: build-system-opensuse
artifacts: true
variables:
IMAGE: opensuse-leap
MAKE_CHECK_ARGS: check
avocado-system-opensuse:
extends: .avocado_test_job_template
needs:
- job: build-system-opensuse
artifacts: true
variables:
IMAGE: opensuse-leap
MAKE_CHECK_ARGS: check-avocado
AVOCADO_TAGS: arch:s390x arch:x86_64 arch:aarch64
#
# Flaky tests. We don't run these by default and they are allow fail
# but often the CI system is the only way to trigger the failures.
#
build-system-flaky:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
job: amd64-debian-container
variables:
IMAGE: debian
QEMU_JOB_OPTIONAL: 1
TARGETS: aarch64-softmmu arm-softmmu mips64el-softmmu
ppc64-softmmu rx-softmmu s390x-softmmu sh4-softmmu x86_64-softmmu
MAKE_CHECK_ARGS: check-build
avocado-system-flaky:
extends: .avocado_test_job_template
needs:
- job: build-system-flaky
artifacts: true
allow_failure: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check-avocado
QEMU_JOB_OPTIONAL: 1
QEMU_TEST_FLAKY_TESTS: 1
AVOCADO_TAGS: flaky
# This jobs explicitly disable TCG (--disable-tcg), KVM is detected by
# the configure script. The container doesn't contain Xen headers so
# Xen accelerator is not detected / selected. As result it build the
# i386-softmmu and x86_64-softmmu with KVM being the single accelerator
# available.
# Also use a different coroutine implementation (which is only really of
# interest to KVM users, i.e. with TCG disabled)
build-tcg-disabled:
extends: .native_build_job_template
needs:
job: amd64-centos8-container
variables:
IMAGE: centos8
script:
- mkdir build
- cd build
- ../configure --disable-tcg --audio-drv-list="" --with-coroutine=ucontext
--disable-docs --disable-sdl --disable-gtk --disable-vnc
|| { cat config.log meson-logs/meson-log.txt && exit 1; }
- make -j"$JOBS"
- make check-unit
- make check-qapi-schema
- cd tests/qemu-iotests/
- ./check -raw 001 002 003 004 005 008 009 010 011 012 021 025 032 033 048
052 063 077 086 101 104 106 113 148 150 151 152 157 159 160 163
170 171 183 184 192 194 208 221 226 227 236 253 277 image-fleecing
- ./check -qcow2 028 051 056 057 058 065 068 082 085 091 095 096 102 122
124 132 139 142 144 145 151 152 155 157 165 194 196 200 202
208 209 216 218 227 234 246 247 248 250 254 255 257 258
260 261 262 263 264 270 272 273 277 279 image-fleecing
build-user:
extends: .native_build_job_template
needs:
job: amd64-debian-user-cross-container
variables:
IMAGE: debian-all-test-cross
CONFIGURE_ARGS: --disable-tools --disable-system
--target-list-exclude=alpha-linux-user,sh4-linux-user
MAKE_CHECK_ARGS: check-tcg
build-user-static:
extends: .native_build_job_template
needs:
job: amd64-debian-user-cross-container
variables:
IMAGE: debian-all-test-cross
CONFIGURE_ARGS: --disable-tools --disable-system --static
--target-list-exclude=alpha-linux-user,sh4-linux-user
MAKE_CHECK_ARGS: check-tcg
# targets stuck on older compilers
build-legacy:
extends: .native_build_job_template
needs:
job: amd64-debian-legacy-cross-container
variables:
IMAGE: debian-legacy-test-cross
TARGETS: alpha-linux-user alpha-softmmu sh4-linux-user
CONFIGURE_ARGS: --disable-tools
MAKE_CHECK_ARGS: check-tcg
build-user-hexagon:
extends: .native_build_job_template
needs:
job: hexagon-cross-container
variables:
IMAGE: debian-hexagon-cross
TARGETS: hexagon-linux-user
CONFIGURE_ARGS: --disable-tools --disable-docs --enable-debug-tcg
MAKE_CHECK_ARGS: check-tcg
# Build the softmmu targets we have check-tcg tests and compilers in
# our omnibus all-test-cross container. Those targets that haven't got
# Debian cross compiler support need to use special containers.
build-some-softmmu:
extends: .native_build_job_template
needs:
job: amd64-debian-user-cross-container
variables:
IMAGE: debian-all-test-cross
CONFIGURE_ARGS: --disable-tools --enable-debug
TARGETS: arm-softmmu aarch64-softmmu i386-softmmu riscv64-softmmu
s390x-softmmu x86_64-softmmu
MAKE_CHECK_ARGS: check-tcg
build-loongarch64:
extends: .native_build_job_template
needs:
job: loongarch-debian-cross-container
variables:
IMAGE: debian-loongarch-cross
CONFIGURE_ARGS: --disable-tools --enable-debug
TARGETS: loongarch64-linux-user loongarch64-softmmu
MAKE_CHECK_ARGS: check-tcg
# We build tricore in a very minimal tricore only container
build-tricore-softmmu:
extends: .native_build_job_template
needs:
job: tricore-debian-cross-container
variables:
IMAGE: debian-tricore-cross
CONFIGURE_ARGS: --disable-tools --disable-fdt --enable-debug
TARGETS: tricore-softmmu
MAKE_CHECK_ARGS: check-tcg
clang-system:
extends: .native_build_job_template
needs:
job: amd64-fedora-container
variables:
IMAGE: fedora
CONFIGURE_ARGS: --cc=clang --cxx=clang++
--extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined
TARGETS: alpha-softmmu arm-softmmu m68k-softmmu mips64-softmmu s390x-softmmu
MAKE_CHECK_ARGS: check-qtest check-tcg
clang-user:
extends: .native_build_job_template
needs:
job: amd64-debian-user-cross-container
timeout: 70m
variables:
IMAGE: debian-all-test-cross
CONFIGURE_ARGS: --cc=clang --cxx=clang++ --disable-system
--target-list-exclude=alpha-linux-user,microblazeel-linux-user,aarch64_be-linux-user,i386-linux-user,m68k-linux-user,mipsn32el-linux-user,xtensaeb-linux-user
--extra-cflags=-fsanitize=undefined --extra-cflags=-fno-sanitize-recover=undefined
MAKE_CHECK_ARGS: check-unit check-tcg
# Set LD_JOBS=1 because this requires LTO and ld consumes a large amount of memory.
# On gitlab runners, default value sometimes end up calling 2 lds concurrently and
# triggers an Out-Of-Memory error
#
# Since slirp callbacks are used in QEMU Timers, we cannot use libslirp with
# CFI builds, and thus have to disable it here.
#
# Split in three sets of build/check/avocado to limit the execution time of each
# job
build-cfi-aarch64:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
- job: amd64-fedora-container
variables:
LD_JOBS: 1
AR: llvm-ar
IMAGE: fedora
CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug
--enable-safe-stack --disable-slirp
TARGETS: aarch64-softmmu
MAKE_CHECK_ARGS: check-build
# FIXME: This job is often failing, likely due to out-of-memory problems in
# the constrained containers of the shared runners. Thus this is marked as
# skipped until the situation has been solved.
QEMU_JOB_SKIPPED: 1
timeout: 90m
check-cfi-aarch64:
extends: .native_test_job_template
needs:
- job: build-cfi-aarch64
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check
avocado-cfi-aarch64:
extends: .avocado_test_job_template
needs:
- job: build-cfi-aarch64
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado
build-cfi-ppc64-s390x:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
- job: amd64-fedora-container
variables:
LD_JOBS: 1
AR: llvm-ar
IMAGE: fedora
CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug
--enable-safe-stack --disable-slirp
TARGETS: ppc64-softmmu s390x-softmmu
MAKE_CHECK_ARGS: check-build
# FIXME: This job is often failing, likely due to out-of-memory problems in
# the constrained containers of the shared runners. Thus this is marked as
# skipped until the situation has been solved.
QEMU_JOB_SKIPPED: 1
timeout: 80m
check-cfi-ppc64-s390x:
extends: .native_test_job_template
needs:
- job: build-cfi-ppc64-s390x
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check
avocado-cfi-ppc64-s390x:
extends: .avocado_test_job_template
needs:
- job: build-cfi-ppc64-s390x
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado
build-cfi-x86_64:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
- job: amd64-fedora-container
variables:
LD_JOBS: 1
AR: llvm-ar
IMAGE: fedora
CONFIGURE_ARGS: --cc=clang --cxx=clang++ --enable-cfi --enable-cfi-debug
--enable-safe-stack --disable-slirp
TARGETS: x86_64-softmmu
MAKE_CHECK_ARGS: check-build
timeout: 70m
check-cfi-x86_64:
extends: .native_test_job_template
needs:
- job: build-cfi-x86_64
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check
avocado-cfi-x86_64:
extends: .avocado_test_job_template
needs:
- job: build-cfi-x86_64
artifacts: true
variables:
IMAGE: fedora
MAKE_CHECK_ARGS: check-avocado
tsan-build:
extends: .native_build_job_template
needs:
job: amd64-ubuntu2204-container
variables:
IMAGE: ubuntu2204
CONFIGURE_ARGS: --enable-tsan --cc=clang --cxx=clang++
--enable-trace-backends=ust --disable-slirp
TARGETS: x86_64-softmmu ppc64-softmmu riscv64-softmmu x86_64-linux-user
# gcov is a GCC features
gcov:
extends: .native_build_job_template
needs:
job: amd64-ubuntu2204-container
timeout: 80m
variables:
IMAGE: ubuntu2204
CONFIGURE_ARGS: --enable-gcov
TARGETS: aarch64-softmmu ppc64-softmmu s390x-softmmu x86_64-softmmu
MAKE_CHECK_ARGS: check-unit check-softfloat
after_script:
- cd build
- gcovr --xml-pretty --exclude-unreachable-branches --print-summary
-o coverage.xml --root ${CI_PROJECT_DIR} . *.p
coverage: /^\s*lines:\s*\d+.\d+\%/
artifacts:
name: ${CI_JOB_NAME}-${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHA}
when: always
expire_in: 2 days
paths:
- build/meson-logs/testlog.txt
reports:
junit: build/meson-logs/testlog.junit.xml
coverage_report:
coverage_format: cobertura
path: build/coverage.xml
build-oss-fuzz:
extends: .native_build_job_template
needs:
job: amd64-fedora-container
variables:
IMAGE: fedora
script:
- mkdir build-oss-fuzz
- export LSAN_OPTIONS=suppressions=scripts/oss-fuzz/lsan_suppressions.txt
- CC="clang" CXX="clang++" CFLAGS="-fsanitize=address"
./scripts/oss-fuzz/build.sh
- export ASAN_OPTIONS="fast_unwind_on_malloc=0"
- for fuzzer in $(find ./build-oss-fuzz/DEST_DIR/ -executable -type f
| grep -v slirp); do
grep "LLVMFuzzerTestOneInput" ${fuzzer} > /dev/null 2>&1 || continue ;
echo Testing ${fuzzer} ... ;
"${fuzzer}" -runs=1 -seed=1 || exit 1 ;
done
build-tci:
extends: .native_build_job_template
needs:
job: amd64-debian-user-cross-container
variables:
IMAGE: debian-all-test-cross
script:
- TARGETS="aarch64 arm hppa m68k microblaze ppc64 s390x x86_64"
- mkdir build
- cd build
- ../configure --enable-tcg-interpreter --disable-docs --disable-gtk --disable-vnc
--target-list="$(for tg in $TARGETS; do echo -n ${tg}'-softmmu '; done)"
|| { cat config.log meson-logs/meson-log.txt && exit 1; }
- make -j"$JOBS"
- make tests/qtest/boot-serial-test tests/qtest/cdrom-test tests/qtest/pxe-test
- for tg in $TARGETS ; do
export QTEST_QEMU_BINARY="./qemu-system-${tg}" ;
./tests/qtest/boot-serial-test || exit 1 ;
./tests/qtest/cdrom-test || exit 1 ;
done
- QTEST_QEMU_BINARY="./qemu-system-x86_64" ./tests/qtest/pxe-test
- QTEST_QEMU_BINARY="./qemu-system-s390x" ./tests/qtest/pxe-test -m slow
- make check-tcg
# Check our reduced build configurations
build-without-defaults:
extends: .native_build_job_template
needs:
job: amd64-centos8-container
variables:
IMAGE: centos8
CONFIGURE_ARGS:
--without-default-devices
--without-default-features
--disable-fdt
--disable-pie
--disable-qom-cast-debug
--disable-strip
TARGETS: avr-softmmu mips64-softmmu s390x-softmmu sh4-softmmu
sparc64-softmmu hexagon-linux-user i386-linux-user s390x-linux-user
MAKE_CHECK_ARGS: check
build-libvhost-user:
extends: .base_job_template
stage: build
image: $CI_REGISTRY_IMAGE/qemu/fedora:$QEMU_CI_CONTAINER_TAG
needs:
job: amd64-fedora-container
script:
- mkdir subprojects/libvhost-user/build
- cd subprojects/libvhost-user/build
- meson
- ninja
# No targets are built here, just tools, docs, and unit tests. This
# also feeds into the eventual documentation deployment steps later
build-tools-and-docs-debian:
extends:
- .native_build_job_template
- .native_build_artifact_template
needs:
job: amd64-debian-container
# when running on 'master' we use pre-existing container
optional: true
variables:
IMAGE: debian
MAKE_CHECK_ARGS: check-unit ctags TAGS cscope
CONFIGURE_ARGS: --disable-system --disable-user --enable-docs --enable-tools
QEMU_JOB_PUBLISH: 1
# Prepare for GitLab pages deployment. Anything copied into the
# "public" directory will be deployed to $USER.gitlab.io/$PROJECT
#
# GitLab publishes from any branch that triggers a CI pipeline
#
# For the main repo we don't want to publish from 'staging'
# since that content may not be pushed, nor do we wish to
# publish from 'stable-NNN' branches as that content is outdated.
# Thus we restrict to just the default branch
#
# For contributor forks we want to publish from any repo so
# that users can see the results of their commits, regardless
# of what topic branch they're currently using
pages:
extends: .base_job_template
image: $CI_REGISTRY_IMAGE/qemu/debian:$QEMU_CI_CONTAINER_TAG
stage: test
needs:
- job: build-tools-and-docs-debian
script:
- mkdir -p public
# HTML-ised source tree
- make gtags
- htags -anT --tree-view=filetree -m qemu_init
-t "Welcome to the QEMU sourcecode"
- mv HTML public/src
# Project documentation
- make -C build install DESTDIR=$(pwd)/temp-install
- mv temp-install/usr/local/share/doc/qemu/* public/
artifacts:
when: on_success
paths:
- public
variables:
QEMU_JOB_PUBLISH: 1