mirrors/qemu
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Integration testing patches

Browse Source 
Tests added:
 - Armbian 20.08 on Orange Pi PC (Philippe)
 - MPC8544ds machine (Thomas)
 - Virtex-ml507 ppc machine (Thomas)
 - Re-enable the microblaze test (Thomas)
 
 Various fixes and documentation improvements from Cleber.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEE+qvnXhKRciHc/Wuy4+MsLN6twN4FAmAhnMAACgkQ4+MsLN6t
 wN4TuA/+Lsk5Sxpe5+3eEaXUAo0v9jHm0Nqh+fbvf/aGkcIbNh0Un3eNpLiv44vE
 vdJlFsfWvwomHylyjPnINBnBjbdIHCC0KdHl6k1YEHVONWpL8t0LyT/eQNXnH4lO
 I6m1uhqR6L5bBIl8gyaQ4PNvDcKtADwqCQUSjuMpSjLzuiwbiwI4Jw137blq9pEm
 zf+LEsD+AZ0zWA1q1bOkRkHDEXaorcUuPGv/es7Xy3tELoXTEBNLEOcu/hNcwimE
 C+X6fyJPCkUxicCRN4X29MEXUgBJin070f/0fxUO3BGpnx1N2lSK5uJwUQ1FilSP
 yPZ6wu9+RakvABuuWlO/mKRR6y9lPhTM2o1niL/hqrVN9DaIysFNYGXosJZ7Cmwq
 KTwcU8fWBT6/66L+4cKxRO7iUHRHeedu+tIqXEjMHkpBpMe7mrVVmcORp8i12Eca
 JVCpjedrX+7X/tWqUKSf63UJrTSFdsdUsdlxWd6uHl5PISJkMWwPsOKxHdMmASqr
 dJmSmCSZ/Gz6+LrmJcB/5/on6NjkiuQ/wFJIkFF1ekhwXJ1Pg/POd1yB7GJd3GZJ
 6XzB2gwNVn3lcuL6kBLHeInUXqDvNAApuLfjHprWQ5N469hCpbXobSqP7Bmvu9Ei
 I56V6EOeLZYb3MYBhMGETz7mJ+zsanYisBPO7gL0ApuPwTQDNdk=
 =mUpx
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/philmd-gitlab/tags/integration-testing-20210208' into staging

Integration testing patches

Tests added:
- Armbian 20.08 on Orange Pi PC (Philippe)
- MPC8544ds machine (Thomas)
- Virtex-ml507 ppc machine (Thomas)
- Re-enable the microblaze test (Thomas)

Various fixes and documentation improvements from Cleber.

# gpg: Signature made Mon 08 Feb 2021 20:19:12 GMT
# gpg:                using RSA key FAABE75E12917221DCFD6BB2E3E32C2CDEADC0DE
# gpg: Good signature from "Philippe Mathieu-Daudé (F4BUG) <f4bug@amsat.org>" [full]
# Primary key fingerprint: FAAB E75E 1291 7221 DCFD  6BB2 E3E3 2C2C DEAD C0DE

* remotes/philmd-gitlab/tags/integration-testing-20210208:
  Acceptance Tests: remove unnecessary tag from documentation example
  Acceptance tests: clarify ssh connection failure reason
  tests/acceptance/virtiofs_submounts: required space between IP and port
  tests/acceptance/virtiofs_submounts: standardize port as integer
  tests/acceptance/virtiofs_submounts: use a virtio-net device instead
  tests/acceptance/virtiofs_submounts: do not ask for ssh key password
  tests/acceptance/virtiofs_submounts: use workdir property
  tests/acceptance/boot_linux: rename misleading cloudinit method
  tests/acceptance/boot_linux: fix typo on cloudinit error message
  tests/acceptance: Re-enable the microblaze test
  tests/acceptance: Add a test for the virtex-ml507 ppc machine
  tests/acceptance: Test the mpc8544ds machine
  tests/acceptance: Move the pseries test to a separate file
  tests/acceptance: Test U-Boot/Linux from Armbian 20.08 on Orange Pi PC
  tests/acceptance: Extract do_test_arm_orangepi_armbian_uboot() method
  tests/acceptance: Introduce tesseract_ocr() helper
  tests/acceptance: Extract tesseract_available() helper in new namespace

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell 2021-02-08 20:22:54 +00:00
commit 34b7d4193e
10 changed files with 226 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
MAINTAINERS
View File

@ -281,6 +281,7 @@ F: target/ppc/
F: hw/ppc/
F: include/hw/ppc/
F: disas/ppc.c
F: tests/acceptance/machine_ppc.py
RISC-V TCG CPUs
M: Palmer Dabbelt <palmer@dabbelt.com>
@ -1118,6 +1119,7 @@ M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
S: Maintained
F: hw/microblaze/petalogix_s3adsp1800_mmu.c
F: include/hw/char/xilinx_uartlite.h
F: tests/acceptance/machine_microblaze.py
petalogix_ml605
M: Edgar E. Iglesias <edgar.iglesias@gmail.com>

3
docs/devel/testing.rst
View File

@ -765,9 +765,6 @@ and hypothetical example follows:
class MultipleMachines(Test):
"""
:avocado: enable
"""
def test_multiple_machines(self):
first_machine = self.get_vm()
second_machine = self.get_vm()

14
tests/acceptance/boot_linux.py
View File

@ -57,7 +57,7 @@ class BootLinuxBase(Test):
self.cancel('Failed to download/prepare boot image')
return boot.path
def download_cloudinit(self, ssh_pubkey=None):
def prepare_cloudinit(self, ssh_pubkey=None):
self.log.info('Preparing cloudinit image')
try:
cloudinit_iso = os.path.join(self.workdir, 'cloudinit.iso')
@ -70,7 +70,7 @@ class BootLinuxBase(Test):
phone_home_port=self.phone_home_port,
authorized_key=ssh_pubkey)
except Exception:
self.cancel('Failed to prepared cloudinit image')
self.cancel('Failed to prepare the cloudinit image')
return cloudinit_iso
class BootLinux(BootLinuxBase):
@ -85,15 +85,15 @@ class BootLinux(BootLinuxBase):
super(BootLinux, self).setUp()
self.vm.add_args('-smp', '2')
self.vm.add_args('-m', '1024')
self.prepare_boot()
self.prepare_cloudinit(ssh_pubkey)
self.set_up_boot()
self.set_up_cloudinit(ssh_pubkey)
def prepare_boot(self):
def set_up_boot(self):
path = self.download_boot()
self.vm.add_args('-drive', 'file=%s' % path)
def prepare_cloudinit(self, ssh_pubkey=None):
cloudinit_iso = self.download_cloudinit(ssh_pubkey)
def set_up_cloudinit(self, ssh_pubkey=None):
cloudinit_iso = self.prepare_cloudinit(ssh_pubkey)
self.vm.add_args('-drive', 'file=%s,format=raw' % cloudinit_iso)
def launch_and_wait(self):

98
tests/acceptance/boot_linux_console.py
View File

@ -802,27 +802,7 @@ class BootLinuxConsole(LinuxKernelTest):
# Wait for VM to shut down gracefully
self.vm.wait()
@skipUnless(os.getenv('ARMBIAN_ARTIFACTS_CACHED'),
'Test artifacts fetched from unreliable dl.armbian.com')
@skipUnless(os.getenv('AVOCADO_ALLOW_LARGE_STORAGE'), 'storage limited')
@skipUnless(P7ZIP_AVAILABLE, '7z not installed')
def test_arm_orangepi_bionic(self):
"""
:avocado: tags=arch:arm
:avocado: tags=machine:orangepi-pc
:avocado: tags=device:sd
"""
# This test download a 196MB compressed image and expand it to 1GB
image_url = ('https://dl.armbian.com/orangepipc/archive/'
'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.7z')
image_hash = '196a8ffb72b0123d92cea4a070894813d305c71e'
image_path_7z = self.fetch_asset(image_url, asset_hash=image_hash)
image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img'
image_path = os.path.join(self.workdir, image_name)
process.run("7z e -o%s %s" % (self.workdir, image_path_7z))
image_pow2ceil_expand(image_path)
def do_test_arm_orangepi_uboot_armbian(self, image_path):
self.vm.set_console()
self.vm.add_args('-drive', 'file=' + image_path + ',if=sd,format=raw',
'-nic', 'user',
@ -848,6 +828,54 @@ class BootLinuxConsole(LinuxKernelTest):
'to <orangepipc>')
self.wait_for_console_pattern('Starting Load Kernel Modules...')
@skipUnless(os.getenv('ARMBIAN_ARTIFACTS_CACHED'),
'Test artifacts fetched from unreliable apt.armbian.com')
@skipUnless(os.getenv('AVOCADO_ALLOW_LARGE_STORAGE'), 'storage limited')
@skipUnless(P7ZIP_AVAILABLE, '7z not installed')
def test_arm_orangepi_bionic_19_11(self):
"""
:avocado: tags=arch:arm
:avocado: tags=machine:orangepi-pc
:avocado: tags=device:sd
"""
# This test download a 196MB compressed image and expand it to 1GB
image_url = ('https://dl.armbian.com/orangepipc/archive/'
'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.7z')
image_hash = '196a8ffb72b0123d92cea4a070894813d305c71e'
image_path_7z = self.fetch_asset(image_url, asset_hash=image_hash)
image_name = 'Armbian_19.11.3_Orangepipc_bionic_current_5.3.9.img'
image_path = os.path.join(self.workdir, image_name)
process.run("7z e -o%s %s" % (self.workdir, image_path_7z))
image_pow2ceil_expand(image_path)
self.do_test_arm_orangepi_uboot_armbian(image_path)
@skipUnless(os.getenv('ARMBIAN_ARTIFACTS_CACHED'),
'Test artifacts fetched from unreliable apt.armbian.com')
@skipUnless(os.getenv('AVOCADO_ALLOW_LARGE_STORAGE'), 'storage limited')
def test_arm_orangepi_bionic_20_08(self):
"""
:avocado: tags=arch:arm
:avocado: tags=machine:orangepi-pc
:avocado: tags=device:sd
"""
# This test download a 275 MiB compressed image and expand it
# to 1036 MiB, but the underlying filesystem is 1552 MiB...
# As we expand it to 2 GiB we are safe.
image_url = ('https://dl.armbian.com/orangepipc/archive/'
'Armbian_20.08.1_Orangepipc_bionic_current_5.8.5.img.xz')
image_hash = ('b4d6775f5673486329e45a0586bf06b6'
'dbe792199fd182ac6b9c7bb6c7d3e6dd')
image_path_xz = self.fetch_asset(image_url, asset_hash=image_hash,
algorithm='sha256')
image_path = archive.extract(image_path_xz, self.workdir)
image_pow2ceil_expand(image_path)
self.do_test_arm_orangepi_uboot_armbian(image_path)
@skipUnless(os.getenv('AVOCADO_ALLOW_LARGE_STORAGE'), 'storage limited')
def test_arm_orangepi_uboot_netbsd9(self):
"""
@ -976,25 +1004,6 @@ class BootLinuxConsole(LinuxKernelTest):
console_pattern = 'Kernel command line: %s' % kernel_command_line
self.wait_for_console_pattern(console_pattern)
def test_ppc64_pseries(self):
"""
:avocado: tags=arch:ppc64
:avocado: tags=machine:pseries
"""
kernel_url = ('https://archives.fedoraproject.org/pub/archive'
'/fedora-secondary/releases/29/Everything/ppc64le/os'
'/ppc/ppc64/vmlinuz')
kernel_hash = '3fe04abfc852b66653b8c3c897a59a689270bc77'
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
self.vm.set_console()
kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=hvc0'
self.vm.add_args('-kernel', kernel_path,
'-append', kernel_command_line)
self.vm.launch()
console_pattern = 'Kernel command line: %s' % kernel_command_line
self.wait_for_console_pattern(console_pattern)
def test_m68k_q800(self):
"""
:avocado: tags=arch:m68k
@ -1047,15 +1056,6 @@ class BootLinuxConsole(LinuxKernelTest):
tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c'
self.do_test_advcal_2018('07', tar_hash, 'sanity-clause.elf')
@skip("Test currently broken") # Console stuck as of 5.2-rc1
def test_microblaze_s3adsp1800(self):
"""
:avocado: tags=arch:microblaze
:avocado: tags=machine:petalogix-s3adsp1800
"""
tar_hash = '08bf3e3bfb6b6c7ce1e54ab65d54e189f2caf13f'
self.do_test_advcal_2018('17', tar_hash, 'ballerina.bin')
def test_or1k_sim(self):
"""
:avocado: tags=arch:or1k

2
tests/acceptance/linux_ssh_mips_malta.py
View File

@ -91,7 +91,7 @@ class LinuxSSH(Test):
except:
time.sleep(4)
pass
self.fail("sshd timeout")
self.fail("ssh connection timeout")
def ssh_disconnect_vm(self):
self.ssh_session.quit()

44
tests/acceptance/machine_m68k_nextcube.py
View File

@ -1,19 +1,17 @@
# Functional test that boots a VM and run OCR on the framebuffer
#
# Copyright (c) Philippe Mathieu-Daudé <f4bug@amsat.org>
# Copyright (c) 2019 Philippe Mathieu-Daudé <f4bug@amsat.org>
#
# This work is licensed under the terms of the GNU GPL, version 2 or
# later. See the COPYING file in the top-level directory.
import os
import re
import time
import logging
from avocado_qemu import Test
from avocado import skipUnless
from avocado.utils import process
from avocado.utils.path import find_command, CmdNotFoundError
from tesseract_utils import tesseract_available, tesseract_ocr
PIL_AVAILABLE = True
try:
@ -22,25 +20,6 @@ except ImportError:
PIL_AVAILABLE = False
def tesseract_available(expected_version):
try:
find_command('tesseract')
except CmdNotFoundError:
return False
res = process.run('tesseract --version')
try:
version = res.stdout_text.split()[1]
except IndexError:
version = res.stderr_text.split()[1]
return int(version.split('.')[0]) == expected_version
match = re.match(r'tesseract\s(\d)', res)
if match is None:
return False
# now this is guaranteed to be a digit
return int(match.groups()[0]) == expected_version
class NextCubeMachine(Test):
"""
:avocado: tags=arch:m68k
@ -80,12 +59,8 @@ class NextCubeMachine(Test):
def test_bootrom_framebuffer_ocr_with_tesseract_v3(self):
screenshot_path = os.path.join(self.workdir, "dump.ppm")
self.check_bootrom_framebuffer(screenshot_path)
console_logger = logging.getLogger('console')
text = process.run("tesseract %s stdout" % screenshot_path).stdout_text
for line in text.split('\n'):
if len(line):
console_logger.debug(line)
lines = tesseract_ocr(screenshot_path, tesseract_version=3)
text = '\n'.join(lines)
self.assertIn('Backplane', text)
self.assertIn('Ethernet address', text)
@ -96,13 +71,8 @@ class NextCubeMachine(Test):
def test_bootrom_framebuffer_ocr_with_tesseract_v4(self):
screenshot_path = os.path.join(self.workdir, "dump.ppm")
self.check_bootrom_framebuffer(screenshot_path)
console_logger = logging.getLogger('console')
proc = process.run("tesseract --oem 1 %s stdout" % screenshot_path)
text = proc.stdout_text
for line in text.split('\n'):
if len(line):
console_logger.debug(line)
lines = tesseract_ocr(screenshot_path, tesseract_version=4)
text = '\n'.join(lines)
self.assertIn('Testing the FPU, SCC', text)
self.assertIn('System test failed. Error code', text)
self.assertIn('Boot command', text)

35
tests/acceptance/machine_microblaze.py Normal file
View File

@ -0,0 +1,35 @@
# Functional test that boots a microblaze Linux kernel and checks the console
#
# Copyright (c) 2018, 2021 Red Hat, Inc.
#
# This work is licensed under the terms of the GNU GPL, version 2 or
# later. See the COPYING file in the top-level directory.
from avocado_qemu import Test
from avocado_qemu import wait_for_console_pattern
from avocado.utils import archive
class MicroblazeMachine(Test):
timeout = 90
def test_microblaze_s3adsp1800(self):
"""
:avocado: tags=arch:microblaze
:avocado: tags=machine:petalogix-s3adsp1800
"""
tar_url = ('https://www.qemu-advent-calendar.org'
'/2018/download/day17.tar.xz')
tar_hash = '08bf3e3bfb6b6c7ce1e54ab65d54e189f2caf13f'
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
archive.extract(file_path, self.workdir)
self.vm.set_console()
self.vm.add_args('-kernel', self.workdir + '/day17/ballerina.bin')
self.vm.launch()
wait_for_console_pattern(self, 'This architecture does not have '
'kernel memory protection')
# Note:
# The kernel sometimes gets stuck after the "This architecture ..."
# message, that's why we don't test for a later string here. This
# needs some investigation by a microblaze wizard one day...

69
tests/acceptance/machine_ppc.py Normal file
View File

@ -0,0 +1,69 @@
# Test that Linux kernel boots on ppc machines and check the console
#
# Copyright (c) 2018, 2020 Red Hat, Inc.
#
# This work is licensed under the terms of the GNU GPL, version 2 or
# later. See the COPYING file in the top-level directory.
from avocado.utils import archive
from avocado_qemu import Test
from avocado_qemu import wait_for_console_pattern
class PpcMachine(Test):
timeout = 90
KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
panic_message = 'Kernel panic - not syncing'
def test_ppc64_pseries(self):
"""
:avocado: tags=arch:ppc64
:avocado: tags=machine:pseries
"""
kernel_url = ('https://archives.fedoraproject.org/pub/archive'
'/fedora-secondary/releases/29/Everything/ppc64le/os'
'/ppc/ppc64/vmlinuz')
kernel_hash = '3fe04abfc852b66653b8c3c897a59a689270bc77'
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
self.vm.set_console()
kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=hvc0'
self.vm.add_args('-kernel', kernel_path,
'-append', kernel_command_line)
self.vm.launch()
console_pattern = 'Kernel command line: %s' % kernel_command_line
wait_for_console_pattern(self, console_pattern, self.panic_message)
def test_ppc_mpc8544ds(self):
"""
:avocado: tags=arch:ppc
:avocado: tags=machine:mpc8544ds
"""
tar_url = ('https://www.qemu-advent-calendar.org'
'/2020/download/day17.tar.gz')
tar_hash = '7a5239542a7c4257aa4d3b7f6ddf08fb6775c494'
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
archive.extract(file_path, self.workdir)
self.vm.set_console()
self.vm.add_args('-kernel', self.workdir + '/creek/creek.bin')
self.vm.launch()
wait_for_console_pattern(self, 'QEMU advent calendar 2020',
self.panic_message)
def test_ppc_virtex_ml507(self):
"""
:avocado: tags=arch:ppc
:avocado: tags=machine:virtex-ml507
"""
tar_url = ('https://www.qemu-advent-calendar.org'
'/2020/download/hippo.tar.gz')
tar_hash = '306b95bfe7d147f125aa176a877e266db8ef914a'
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
archive.extract(file_path, self.workdir)
self.vm.set_console()
self.vm.add_args('-kernel', self.workdir + '/hippo/hippo.linux',
'-dtb', self.workdir + '/hippo/virtex440-ml507.dtb',
'-m', '512')
self.vm.launch()
wait_for_console_pattern(self, 'QEMU advent calendar 2020',
self.panic_message)

46
tests/acceptance/tesseract_utils.py Normal file
View File

@ -0,0 +1,46 @@
# ...
#
# Copyright (c) 2019 Philippe Mathieu-Daudé <f4bug@amsat.org>
#
# This work is licensed under the terms of the GNU GPL, version 2 or
# later. See the COPYING file in the top-level directory.
import re
import logging
from avocado.utils import process
from avocado.utils.path import find_command, CmdNotFoundError
def tesseract_available(expected_version):
try:
find_command('tesseract')
except CmdNotFoundError:
return False
res = process.run('tesseract --version')
try:
version = res.stdout_text.split()[1]
except IndexError:
version = res.stderr_text.split()[1]
return int(version.split('.')[0]) == expected_version
match = re.match(r'tesseract\s(\d)', res)
if match is None:
return False
# now this is guaranteed to be a digit
return int(match.groups()[0]) == expected_version
def tesseract_ocr(image_path, tesseract_args='', tesseract_version=3):
console_logger = logging.getLogger('tesseract')
console_logger.debug(image_path)
if tesseract_version == 4:
tesseract_args += ' --oem 1'
proc = process.run("tesseract {} {} stdout".format(tesseract_args,
image_path))
lines = []
for line in proc.stdout_text.split('\n'):
sline = line.strip()
if len(sline):
console_logger.debug(sline)
lines += [sline]
return lines

21
tests/acceptance/virtiofs_submounts.py
View File

@ -83,20 +83,21 @@ class VirtiofsSubmountsTest(BootLinux):
command_line='info usernet')
for line in res.split('\r\n'):
match = \
re.search(r'TCP.HOST_FORWARD.*127\.0\.0\.1\s*(\d+)\s+10\.',
re.search(r'TCP.HOST_FORWARD.*127\.0\.0\.1\s+(\d+)\s+10\.',
line)
if match is not None:
port = match[1]
port = int(match[1])
break
self.assertIsNotNone(port)
self.log.debug('sshd listening on port: ' + port)
self.assertGreater(port, 0)
self.log.debug('sshd listening on port: %d', port)
return port
def ssh_connect(self, username, keyfile):
self.ssh_logger = logging.getLogger('ssh')
port = self.get_portfwd()
self.ssh_session = ssh.Session('127.0.0.1', port=int(port),
self.ssh_session = ssh.Session('127.0.0.1', port=port,
user=username, key=keyfile)
for i in range(10):
try:
@ -105,7 +106,7 @@ class VirtiofsSubmountsTest(BootLinux):
except:
time.sleep(4)
pass
self.fail('sshd timeout')
self.fail('ssh connection timeout')
def ssh_command(self, command):
self.ssh_logger.info(command)
@ -136,8 +137,7 @@ class VirtiofsSubmountsTest(BootLinux):
return (stdout, stderr, ret)
def set_up_shared_dir(self):
atwd = os.getenv('AVOCADO_TEST_WORKDIR')
self.shared_dir = os.path.join(atwd, 'virtiofs-shared')
self.shared_dir = os.path.join(self.workdir, 'virtiofs-shared')
os.mkdir(self.shared_dir)
@ -234,10 +234,9 @@ class VirtiofsSubmountsTest(BootLinux):
self.seed = self.params.get('seed')
atwd = os.getenv('AVOCADO_TEST_WORKDIR')
self.ssh_key = os.path.join(atwd, 'id_ed25519')
self.ssh_key = os.path.join(self.workdir, 'id_ed25519')
self.run(('ssh-keygen', '-t', 'ed25519', '-f', self.ssh_key))
self.run(('ssh-keygen', '-N', '', '-t', 'ed25519', '-f', self.ssh_key))
pubkey = open(self.ssh_key + '.pub').read()
@ -249,7 +248,7 @@ class VirtiofsSubmountsTest(BootLinux):
# Allow us to connect to SSH
self.vm.add_args('-netdev', 'user,id=vnet,hostfwd=:127.0.0.1:0-:22',
'-device', 'e1000,netdev=vnet')
'-device', 'virtio-net,netdev=vnet')
if not kvm_available(self.arch, self.qemu_bin):
self.cancel(KVM_NOT_AVAILABLE)