qemu/.gitlab-ci.d/buildtest-template.yml

129 lines
3.9 KiB
YAML
Raw Normal View History

.native_build_job_template:
extends: .base_job_template
stage: build
image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
gitlab: enable ccache for many build jobs The `ccache` tool can be very effective at reducing compilation times when re-running pipelines with only minor changes each time. For example a fresh 'build-system-fedora' job will typically take 20 minutes on the gitlab.com shared runners. With ccache this is reduced to as little as 6 minutes. Normally meson would auto-detect existance of ccache in $PATH and use it automatically, but the way we wrap meson from configure breaks this, as we're passing in an config file with explicitly set compiler paths. Thus we need to add $CCACHE_WRAPPERSPATH to the front of $PATH. For unknown reasons if doing this in msys though, gcc becomes unable to invoke 'cc1' when run from meson. For msys we thus set CC='ccache gcc' before invoking 'configure' instead. A second problem with msys is that cache misses are incredibly expensive, so enabling ccache massively slows down the build when the cache isn't well populated. This is suspected to be a result of the cost of spawning processes under the msys architecture. To deal with this we set CCACHE_DEPEND=1 which enables ccache's 'depend_only' strategy. This avoids extra spawning of the pre-processor during cache misses, with the downside that is it less likely ccache will find a cache hit after semantically benign compiler flag changes. This is the lesser of two evils, as otherwise we can't use ccache at all under msys and remain inside the job time limit. If people are finding ccache to hurt their pipelines, it can be disabled by setting the 'CCACHE_DISABLE=1' env variable against their gitlab fork CI settings. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Message-Id: <20230804111054.281802-2-berrange@redhat.com> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Message-Id: <20230829161528.2707696-2-alex.bennee@linaro.org>
2023-08-29 19:15:17 +03:00
cache:
paths:
- ccache
key: "$CI_JOB_NAME"
when: always
before_script:
- source scripts/ci/gitlab-ci-section
- section_start setup "Pre-script setup"
- JOBS=$(expr $(nproc) + 1)
- cat /packages.txt
- section_end setup
script:
gitlab: enable ccache for many build jobs The `ccache` tool can be very effective at reducing compilation times when re-running pipelines with only minor changes each time. For example a fresh 'build-system-fedora' job will typically take 20 minutes on the gitlab.com shared runners. With ccache this is reduced to as little as 6 minutes. Normally meson would auto-detect existance of ccache in $PATH and use it automatically, but the way we wrap meson from configure breaks this, as we're passing in an config file with explicitly set compiler paths. Thus we need to add $CCACHE_WRAPPERSPATH to the front of $PATH. For unknown reasons if doing this in msys though, gcc becomes unable to invoke 'cc1' when run from meson. For msys we thus set CC='ccache gcc' before invoking 'configure' instead. A second problem with msys is that cache misses are incredibly expensive, so enabling ccache massively slows down the build when the cache isn't well populated. This is suspected to be a result of the cost of spawning processes under the msys architecture. To deal with this we set CCACHE_DEPEND=1 which enables ccache's 'depend_only' strategy. This avoids extra spawning of the pre-processor during cache misses, with the downside that is it less likely ccache will find a cache hit after semantically benign compiler flag changes. This is the lesser of two evils, as otherwise we can't use ccache at all under msys and remain inside the job time limit. If people are finding ccache to hurt their pipelines, it can be disabled by setting the 'CCACHE_DISABLE=1' env variable against their gitlab fork CI settings. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Message-Id: <20230804111054.281802-2-berrange@redhat.com> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Message-Id: <20230829161528.2707696-2-alex.bennee@linaro.org>
2023-08-29 19:15:17 +03:00
- export CCACHE_BASEDIR="$(pwd)"
- export CCACHE_DIR="$CCACHE_BASEDIR/ccache"
- export CCACHE_MAXSIZE="500M"
- export PATH="$CCACHE_WRAPPERSDIR:$PATH"
- du -sh .git
- mkdir build
- cd build
gitlab: enable ccache for many build jobs The `ccache` tool can be very effective at reducing compilation times when re-running pipelines with only minor changes each time. For example a fresh 'build-system-fedora' job will typically take 20 minutes on the gitlab.com shared runners. With ccache this is reduced to as little as 6 minutes. Normally meson would auto-detect existance of ccache in $PATH and use it automatically, but the way we wrap meson from configure breaks this, as we're passing in an config file with explicitly set compiler paths. Thus we need to add $CCACHE_WRAPPERSPATH to the front of $PATH. For unknown reasons if doing this in msys though, gcc becomes unable to invoke 'cc1' when run from meson. For msys we thus set CC='ccache gcc' before invoking 'configure' instead. A second problem with msys is that cache misses are incredibly expensive, so enabling ccache massively slows down the build when the cache isn't well populated. This is suspected to be a result of the cost of spawning processes under the msys architecture. To deal with this we set CCACHE_DEPEND=1 which enables ccache's 'depend_only' strategy. This avoids extra spawning of the pre-processor during cache misses, with the downside that is it less likely ccache will find a cache hit after semantically benign compiler flag changes. This is the lesser of two evils, as otherwise we can't use ccache at all under msys and remain inside the job time limit. If people are finding ccache to hurt their pipelines, it can be disabled by setting the 'CCACHE_DISABLE=1' env variable against their gitlab fork CI settings. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Message-Id: <20230804111054.281802-2-berrange@redhat.com> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Message-Id: <20230829161528.2707696-2-alex.bennee@linaro.org>
2023-08-29 19:15:17 +03:00
- ccache --zero-stats
- section_start configure "Running configure"
- ../configure --enable-werror --disable-docs --enable-fdt=system
configure: use 'mkvenv ensure meson' to bootstrap meson This commit changes how we detect and install meson. It notably removes '--meson='. Currently, configure creates a lightweight Python virtual environment unconditionally using the user's configured $python that inherits system packages. Temporarily, we forced the use of meson source present via git submodule or in the release tarball. With this patch, we restore the ability to use a system-provided meson: If Meson is installed in the build venv and meets our minimum version requirements, we will use that Meson. This includes a system provided meson, which would be visible via system-site packages inside the venv. In the event that Meson is installed but *not for the chosen Python interpreter*, not found, or of insufficient version, we will attempt to install Meson from vendored source into the newly created Python virtual environment. This vendored installation replaces both the git submodule and tarball source mechanisms for sourcing meson. As a result of this patch, the Python interpreter we use for both our own build scripts *and* Meson extensions are always known to be the exact same Python. As a further benefit, there will also be a symlink available in the build directory that points to the correct, configured python and can be used by e.g. manual tests to invoke the correct, configured Python unambiguously. Signed-off-by: John Snow <jsnow@redhat.com> Message-Id: <20230511035435.734312-18-jsnow@redhat.com> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2023-05-11 06:54:25 +03:00
${TARGETS:+--target-list="$TARGETS"}
$CONFIGURE_ARGS ||
{ cat config.log meson-logs/meson-log.txt && exit 1; }
- if test -n "$LD_JOBS";
then
configure: use 'mkvenv ensure meson' to bootstrap meson This commit changes how we detect and install meson. It notably removes '--meson='. Currently, configure creates a lightweight Python virtual environment unconditionally using the user's configured $python that inherits system packages. Temporarily, we forced the use of meson source present via git submodule or in the release tarball. With this patch, we restore the ability to use a system-provided meson: If Meson is installed in the build venv and meets our minimum version requirements, we will use that Meson. This includes a system provided meson, which would be visible via system-site packages inside the venv. In the event that Meson is installed but *not for the chosen Python interpreter*, not found, or of insufficient version, we will attempt to install Meson from vendored source into the newly created Python virtual environment. This vendored installation replaces both the git submodule and tarball source mechanisms for sourcing meson. As a result of this patch, the Python interpreter we use for both our own build scripts *and* Meson extensions are always known to be the exact same Python. As a further benefit, there will also be a symlink available in the build directory that points to the correct, configured python and can be used by e.g. manual tests to invoke the correct, configured Python unambiguously. Signed-off-by: John Snow <jsnow@redhat.com> Message-Id: <20230511035435.734312-18-jsnow@redhat.com> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
2023-05-11 06:54:25 +03:00
pyvenv/bin/meson configure . -Dbackend_max_links="$LD_JOBS" ;
fi || exit 1;
- section_end configure
- section_start build "Building QEMU"
- $MAKE -j"$JOBS"
- section_end build
- section_start test "Running tests"
- if test -n "$MAKE_CHECK_ARGS";
then
$MAKE -j"$JOBS" $MAKE_CHECK_ARGS ;
fi
- section_end test
gitlab: enable ccache for many build jobs The `ccache` tool can be very effective at reducing compilation times when re-running pipelines with only minor changes each time. For example a fresh 'build-system-fedora' job will typically take 20 minutes on the gitlab.com shared runners. With ccache this is reduced to as little as 6 minutes. Normally meson would auto-detect existance of ccache in $PATH and use it automatically, but the way we wrap meson from configure breaks this, as we're passing in an config file with explicitly set compiler paths. Thus we need to add $CCACHE_WRAPPERSPATH to the front of $PATH. For unknown reasons if doing this in msys though, gcc becomes unable to invoke 'cc1' when run from meson. For msys we thus set CC='ccache gcc' before invoking 'configure' instead. A second problem with msys is that cache misses are incredibly expensive, so enabling ccache massively slows down the build when the cache isn't well populated. This is suspected to be a result of the cost of spawning processes under the msys architecture. To deal with this we set CCACHE_DEPEND=1 which enables ccache's 'depend_only' strategy. This avoids extra spawning of the pre-processor during cache misses, with the downside that is it less likely ccache will find a cache hit after semantically benign compiler flag changes. This is the lesser of two evils, as otherwise we can't use ccache at all under msys and remain inside the job time limit. If people are finding ccache to hurt their pipelines, it can be disabled by setting the 'CCACHE_DISABLE=1' env variable against their gitlab fork CI settings. Signed-off-by: Daniel P. Berrangé <berrange@redhat.com> Reviewed-by: Philippe Mathieu-Daudé <philmd@linaro.org> Message-Id: <20230804111054.281802-2-berrange@redhat.com> Signed-off-by: Alex Bennée <alex.bennee@linaro.org> Message-Id: <20230829161528.2707696-2-alex.bennee@linaro.org>
2023-08-29 19:15:17 +03:00
- ccache --show-stats
# We jump some hoops in common_test_job_template to avoid
# rebuilding all the object files we skip in the artifacts
.native_build_artifact_template:
artifacts:
when: on_success
expire_in: 2 days
paths:
- build
- .git-submodule-status
exclude:
- build/**/*.p
- build/**/*.a.p
- build/**/*.c.o
- build/**/*.c.o.d
.common_test_job_template:
extends: .base_job_template
stage: test
image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:$QEMU_CI_CONTAINER_TAG
script:
- source scripts/ci/gitlab-ci-section
- section_start buildenv "Setting up to run tests"
- scripts/git-submodule.sh update roms/SLOF
- meson subprojects download $(cd build/subprojects && echo *)
- cd build
- find . -type f -exec touch {} +
# Avoid recompiling by hiding ninja with NINJA=":"
# We also have to pre-cache the functional tests manually in this case
- if [ "x${QEMU_TEST_CACHE_DIR}" != "x" ]; then
$MAKE precache-functional ;
fi
- section_end buildenv
- section_start test "Running tests"
- $MAKE NINJA=":" $MAKE_CHECK_ARGS
- section_end test
.native_test_job_template:
extends: .common_test_job_template
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
when: always
expire_in: 7 days
paths:
- build/meson-logs/testlog.txt
reports:
junit: build/meson-logs/testlog.junit.xml
.functional_test_job_template:
extends: .common_test_job_template
cache:
key: "${CI_JOB_NAME}-cache"
paths:
- ${CI_PROJECT_DIR}/avocado-cache
- ${CI_PROJECT_DIR}/functional-cache
policy: pull-push
artifacts:
name: "$CI_JOB_NAME-$CI_COMMIT_REF_SLUG"
when: always
expire_in: 7 days
paths:
- build/tests/results/latest/results.xml
- build/tests/results/latest/test-results
- build/tests/functional/*/*/*.log
reports:
junit: build/tests/results/latest/results.xml
before_script:
- mkdir -p ~/.config/avocado
- echo "[datadir.paths]" > ~/.config/avocado/avocado.conf
- echo "cache_dirs = ['${CI_PROJECT_DIR}/avocado-cache']"
>> ~/.config/avocado/avocado.conf
- echo -e '[job.output.testlogs]\nstatuses = ["FAIL", "INTERRUPT"]'
>> ~/.config/avocado/avocado.conf
- if [ -d ${CI_PROJECT_DIR}/avocado-cache ]; then
du -chs ${CI_PROJECT_DIR}/*-cache ;
fi
- export AVOCADO_ALLOW_UNTRUSTED_CODE=1
- export QEMU_TEST_ALLOW_UNTRUSTED_CODE=1
- export QEMU_TEST_CACHE_DIR=${CI_PROJECT_DIR}/functional-cache
after_script:
- cd build
- du -chs ${CI_PROJECT_DIR}/*-cache
variables:
QEMU_JOB_AVOCADO: 1