From 414a8ce57e0b114a8c4cda143e34a3da0b321a23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 20 Feb 2017 10:51:36 +0000 Subject: [PATCH 1/5] tests/docker: add basic user mapping support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently all docker builds are done by exporting a tarball to the docker container and running the build as the containers root user. Other use cases are possible however and it is possible to map a part of users file-system to the container. This is useful for example for doing cross-builds of arbitrary source trees. For this to work smoothly the container needs to have a user created that maps cleanly to the host system. This adds a -u option to the docker script so that: DEB_ARCH=armhf DEB_TYPE=stable ./tests/docker/docker.py build \ -u --include-executable=arm-linux-user/qemu-arm \ debian:armhf ./tests/docker/dockerfiles/debian-bootstrap.docker Will build a container that can then be run like: docker run --rm -it -v /home/alex/lsrc/qemu/risu.git/:/src \ --user=alex:alex -w /src/ debian:armhf \ sh -c "make clean && ./configure -s && make" All docker containers built will add the current user unless explicitly disabled by specifying NOUSER when invoking the Makefile: make docker-image-debian-armhf-cross NOUSER=1 Signed-off-by: Alex Bennée Reviewed-by: Fam Zheng Tested-by: Philippe Mathieu-Daudé Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20170220105139.21581-2-alex.bennee@linaro.org> Signed-off-by: Fam Zheng --- tests/docker/Makefile.include | 2 ++ tests/docker/docker.py | 16 ++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 3f15d5aea8..3b5ffecb04 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -50,6 +50,7 @@ docker-image-%: $(DOCKER_FILES_DIR)/%.docker $(call quiet-command,\ $(SRC_PATH)/tests/docker/docker.py build qemu:$* $< \ $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \ + $(if $(NOUSER),,--add-current-user) \ $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\ "BUILD","$*") @@ -99,6 +100,7 @@ docker: @echo ' (default is 1)' @echo ' DEBUG=1 Stop and drop to shell in the created container' @echo ' before running the command.' + @echo ' NOUSER Define to disable adding current user to containers passwd.' @echo ' NOCACHE=1 Ignore cache when build images.' @echo ' EXECUTABLE= Include executable in image.' diff --git a/tests/docker/docker.py b/tests/docker/docker.py index 37d83199e7..9fd32ab5fa 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -25,6 +25,7 @@ import signal from tarfile import TarFile, TarInfo from StringIO import StringIO from shutil import copy, rmtree +from pwd import getpwuid DEVNULL = open(os.devnull, 'wb') @@ -149,13 +150,21 @@ class Docker(object): labels = json.loads(resp)[0]["Config"].get("Labels", {}) return labels.get("com.qemu.dockerfile-checksum", "") - def build_image(self, tag, docker_dir, dockerfile, quiet=True, argv=None): + def build_image(self, tag, docker_dir, dockerfile, + quiet=True, user=False, argv=None): if argv == None: argv = [] tmp_df = tempfile.NamedTemporaryFile(dir=docker_dir, suffix=".docker") tmp_df.write(dockerfile) + if user: + uid = os.getuid() + uname = getpwuid(uid).pw_name + tmp_df.write("\n") + tmp_df.write("RUN id %s 2>/dev/null || useradd -u %d -U %s" % + (uname, uid, uname)) + tmp_df.write("\n") tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % _text_checksum(dockerfile)) @@ -225,6 +234,9 @@ class BuildCommand(SubCommand): help="""Specify a binary that will be copied to the container together with all its dependent libraries""") + parser.add_argument("--add-current-user", "-u", dest="user", + action="store_true", + help="Add the current user to image's passwd") parser.add_argument("tag", help="Image Tag") parser.add_argument("dockerfile", @@ -261,7 +273,7 @@ class BuildCommand(SubCommand): docker_dir) dkr.build_image(tag, docker_dir, dockerfile, - quiet=args.quiet, argv=argv) + quiet=args.quiet, user=args.user, argv=argv) rmtree(docker_dir) From 24e0131f372b3b3798460c6694d238ddbd32db11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 20 Feb 2017 10:51:37 +0000 Subject: [PATCH 2/5] new: debian docker targets for cross-compiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This provides a basic Debian install with access to the emdebian cross compilers. The debian-armhf-cross and debian-arm64-cross targets build on the basic Debian image to allow cross compiling to those targets. A new environment variable (QEMU_CONFIGURE_OPTS) is set as part of the docker container and passed to the build to specify the --cross-prefix. The user still calls the build in the usual way, for example: make docker-test-build@debian-arm64-cross \ TARGET_LIST="aarch64-softmmu,aarch64-linux-user" Signed-off-by: Alex Bennée Reviewed-by: Fam Zheng Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20170220105139.21581-3-alex.bennee@linaro.org> Signed-off-by: Fam Zheng --- tests/docker/Makefile.include | 4 +++ tests/docker/common.rc | 2 +- .../dockerfiles/debian-arm64-cross.docker | 15 +++++++++++ .../dockerfiles/debian-armhf-cross.docker | 15 +++++++++++ tests/docker/dockerfiles/debian.docker | 25 +++++++++++++++++++ 5 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 tests/docker/dockerfiles/debian-arm64-cross.docker create mode 100644 tests/docker/dockerfiles/debian-armhf-cross.docker create mode 100644 tests/docker/dockerfiles/debian.docker diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 3b5ffecb04..03eda37bf4 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -54,6 +54,10 @@ docker-image-%: $(DOCKER_FILES_DIR)/%.docker $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\ "BUILD","$*") +# Enforce dependancies for composite images +docker-image-debian-armhf-cross: docker-image-debian +docker-image-debian-arm64-cross: docker-image-debian + # Expand all the pre-requistes for each docker image and test combination $(foreach i,$(DOCKER_IMAGES), \ $(foreach t,$(DOCKER_TESTS) $(DOCKER_TOOLS), \ diff --git a/tests/docker/common.rc b/tests/docker/common.rc index 21657e87c6..6865689bb5 100755 --- a/tests/docker/common.rc +++ b/tests/docker/common.rc @@ -29,7 +29,7 @@ build_qemu() config_opts="--enable-werror \ ${TARGET_LIST:+--target-list=${TARGET_LIST}} \ --prefix=$PWD/install \ - $EXTRA_CONFIGURE_OPTS \ + $QEMU_CONFIGURE_OPTS $EXTRA_CONFIGURE_OPTS \ $@" echo "Configure options:" echo $config_opts diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker new file mode 100644 index 0000000000..592b5d7055 --- /dev/null +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -0,0 +1,15 @@ +# +# Docker arm64 cross-compiler target +# +# This docker target builds on the base debian image. +# +FROM qemu:debian + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture arm64 +RUN apt update +RUN apt install -yy crossbuild-essential-arm64 +RUN apt-get build-dep -yy -a arm64 qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=aarch64-linux-gnu- diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker new file mode 100644 index 0000000000..668d60aeb3 --- /dev/null +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -0,0 +1,15 @@ +# +# Docker armhf cross-compiler target +# +# This docker target builds on the base debian image. +# +FROM qemu:debian + +# Add the foreign architecture we want and install dependencies +RUN dpkg --add-architecture armhf +RUN apt update +RUN apt install -yy crossbuild-essential-armhf +RUN apt-get build-dep -yy -a armhf qemu + +# Specify the cross prefix for this image (see tests/docker/common.rc) +ENV QEMU_CONFIGURE_OPTS --cross-prefix=arm-linux-gnueabihf- diff --git a/tests/docker/dockerfiles/debian.docker b/tests/docker/dockerfiles/debian.docker new file mode 100644 index 0000000000..52bd79938e --- /dev/null +++ b/tests/docker/dockerfiles/debian.docker @@ -0,0 +1,25 @@ +# +# Docker multiarch cross-compiler target +# +# This docker target is builds on Debian and Emdebian's cross compiler targets +# to build distro with a selection of cross compilers for building test binaries. +# +# On its own you can't build much but the docker-foo-cross targets +# build on top of the base debian image. +# +FROM debian:stable-slim + +# Setup some basic tools we need +RUN apt update +RUN apt install -yy curl aptitude + +# Setup Emdebian +RUN echo "deb http://emdebian.org/tools/debian/ jessie main" >> /etc/apt/sources.list +RUN curl http://emdebian.org/tools/debian/emdebian-toolchain-archive.key | apt-key add - + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/deb/deb-src/" >> /etc/apt/sources.list + +# Install common build utilities +RUN apt update +RUN apt install -yy build-essential clang From d92d886a3b6daadb79d1635ad281685e80d3ac98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 20 Feb 2017 10:51:38 +0000 Subject: [PATCH 3/5] .shippable.yml: new CI provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ostensibly Shippable offers a similar set of services as Travis. However they are focused on Docker container based work-flows so we can use our existing containers to run a few extra builds - in this case a bunch of cross-compiled targets on a Debian multiarch system. Signed-off-by: Alex Bennée Reviewed-by: Fam Zheng Message-Id: <20170220105139.21581-4-alex.bennee@linaro.org> Signed-off-by: Fam Zheng --- .shippable.yml | 19 +++++++++++++++++++ MAINTAINERS | 1 + 2 files changed, 20 insertions(+) create mode 100644 .shippable.yml diff --git a/.shippable.yml b/.shippable.yml new file mode 100644 index 0000000000..1a1fd7a91d --- /dev/null +++ b/.shippable.yml @@ -0,0 +1,19 @@ +language: c +env: + matrix: + - IMAGE=debian-armhf-cross + TARGET_LIST=arm-softmmu,arm-linux-user + - IMAGE=debian-arm64-cross + TARGET_LIST=aarch64-softmmu,aarch64-linux-user +build: + pre_ci: + - make docker-image-${IMAGE} + pre_ci_boot: + image_name: qemu + image_tag: ${IMAGE} + pull: false + options: "-e HOME=/root" + ci: + - unset CC + - ./configure ${QEMU_CONFIGURE_OPTS} --target-list=${TARGET_LIST} + - make -j2 diff --git a/MAINTAINERS b/MAINTAINERS index 4714df883b..67404677a1 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1803,6 +1803,7 @@ M: Alex Bennée L: qemu-devel@nongnu.org S: Supported F: .travis.yml +F: .shippable.yml Documentation ------------- From e70dc7f854392c7090db279b546b24c6b3ca6567 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Mon, 20 Feb 2017 10:51:39 +0000 Subject: [PATCH 4/5] MAINTAINERS: merge Build and test automation with Docker tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The docker framework is really just another piece in the build automation puzzle so lets merge it together. For added bonus I've also included the Travis and Patchew status links. The Shippable links will be added later once mainline tests have been configured and setup. Signed-off-by: Alex Bennée Reviewed-by: Fam Zheng Message-Id: <20170220105139.21581-5-alex.bennee@linaro.org> Signed-off-by: Fam Zheng --- MAINTAINERS | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 67404677a1..be79f68f46 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1800,10 +1800,14 @@ F: docs/block-replication.txt Build and test automation ------------------------- M: Alex Bennée +M: Fam Zheng L: qemu-devel@nongnu.org -S: Supported +S: Maintained F: .travis.yml F: .shippable.yml +F: tests/docker/ +W: https://travis-ci.org/qemu/qemu +W: http://patchew.org/QEMU/ Documentation ------------- @@ -1812,9 +1816,3 @@ M: Daniel P. Berrange S: Odd Fixes F: docs/build-system.txt -Docker testing --------------- -Docker based testing framework and cases -M: Fam Zheng -S: Maintained -F: tests/docker/ From a8f159d45bcb78c00ea160bfc2b94512e4ed9910 Mon Sep 17 00:00:00 2001 From: Fam Zheng Date: Wed, 22 Feb 2017 10:18:01 +0800 Subject: [PATCH 5/5] docker: Install python2 explicitly in docker image Python is no longer installed implicitly, but the QEMU build system requires it. List it in PACKAGES. Reported-by: Auger Eric Signed-off-by: Fam Zheng Message-Id: <20170222021801.28658-1-famz@redhat.com> Tested-by: Eric Auger Signed-off-by: Fam Zheng --- tests/docker/dockerfiles/fedora.docker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index 478163b8d8..c4f80ad3d8 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -1,6 +1,6 @@ FROM fedora:latest ENV PACKAGES \ - ccache git tar PyYAML sparse flex bison \ + ccache git tar PyYAML sparse flex bison python2 \ glib2-devel pixman-devel zlib-devel SDL-devel libfdt-devel \ gcc gcc-c++ clang make perl which bc findutils \ mingw32-pixman mingw32-glib2 mingw32-gmp mingw32-SDL mingw32-pkg-config \