mirror of https://github.com/xemu-project/xemu.git
Acceptance tests patches
- List acceptance test reviewers in MAINTAINERS - Record/Replay tests from Pavel Dovgalyuk Example of use: $ avocado --show=app,replay run -t machine:vexpress-a9 tests/acceptance/replay_kernel.py Fetching asset from tests/acceptance/replay_kernel.py:ReplayKernel.test_arm_vexpressa9 (1/1) tests/acceptance/replay_kernel.py:ReplayKernel.test_arm_vexpressa9: replay: recording the execution... replay: finished the recording with log size 204784 bytes replay: elapsed time 6.44 sec replay: replaying the execution... replay: successfully finished the replay replay: elapsed time 7.97 sec replay: replay overhead 23.86% PASS (14.67 s) Travis-CI: https://travis-ci.org/github/philmd/qemu/jobs/700787719 -----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEE+qvnXhKRciHc/Wuy4+MsLN6twN4FAl7wcqUACgkQ4+MsLN6t wN4m8w/9H++cGOwr0AxnCKuF0xHLa4appei3GlrY5t6+mChwJZsdmnC4hTHigiNM NMlj6IrlFFi6Ygbf1s8R6D6MoYtVWoGsiRfwWRXN/kvZ6MIDX9zv4cycmUB8mrwV uJ+FX1YecZRdwQNdruWVx/HyqEBjaZ60jGCPKv/8r6cbpFbqzfppgvYsHgcujf6B v9Z9szoUjq+bgs6fPnleg5A2E1PpKmtYebpWYg1eg6oTyqcgjYxoh4jlT8AiJh+z mqoIulyrYRarRrH4qcTCODM9gMVBvssk4Q1GbUJNIm0kwU96W/4uQNcSgHyL96dD TZMues94+syJUxl6AkXrh6o0IrOGtUliDpZKwSCZi0LBxqTRmlqQutDVGDq11iwE 6RgzPjp7t1CHKOPkoNGM3IIFBmVuCj1PelEPCAUNJbxcgv76KCnchB9sSjVdQEFu RHMGYWWIDLqeSOW6O+qtJ+tDrev7Rdfyw2zibHtI1UN9Oaq1q2ChCyPaw70x+U9J XGUS5xci/xB/hv9P7VrfS61kCqrUa5JNvRN3vUKaWIK4KNYBvzOiNMC4oawYNLj3 /yjwziYTa702x8WPK1t7kTBEdiwiZmp0T1y0WOmUqDosWnQbZD15NooTihW83Aq0 CBUV+MUesQ0dxtMc6QdkrRliLwJSVqzZXK4bFp8abLLxAqac/Bw= =KViY -----END PGP SIGNATURE----- Merge remote-tracking branch 'remotes/philmd-gitlab/tags/acceptance-testing-20200622' into staging Acceptance tests patches - List acceptance test reviewers in MAINTAINERS - Record/Replay tests from Pavel Dovgalyuk Example of use: $ avocado --show=app,replay run -t machine:vexpress-a9 tests/acceptance/replay_kernel.py Fetching asset from tests/acceptance/replay_kernel.py:ReplayKernel.test_arm_vexpressa9 (1/1) tests/acceptance/replay_kernel.py:ReplayKernel.test_arm_vexpressa9: replay: recording the execution... replay: finished the recording with log size 204784 bytes replay: elapsed time 6.44 sec replay: replaying the execution... replay: successfully finished the replay replay: elapsed time 7.97 sec replay: replay overhead 23.86% PASS (14.67 s) Travis-CI: https://travis-ci.org/github/philmd/qemu/jobs/700787719 # gpg: Signature made Mon 22 Jun 2020 09:58:13 BST # gpg: using RSA key FAABE75E12917221DCFD6BB2E3E32C2CDEADC0DE # gpg: Good signature from "Philippe Mathieu-Daudé (F4BUG) <f4bug@amsat.org>" [full] # Primary key fingerprint: FAAB E75E 1291 7221 DCFD 6BB2 E3E3 2C2C DEAD C0DE * remotes/philmd-gitlab/tags/acceptance-testing-20200622: tests/acceptance: record/replay tests with advcal images tests/acceptance: add record/replay test for m68k tests/acceptance: add record/replay test for ppc64 tests/acceptance: add record/replay test for arm tests/acceptance: add record/replay test for aarch64 tests/acceptance: add kernel record/replay test for x86_64 tests/acceptance: add base class record/replay kernel tests MAINTAINERS: Add an entry to review Avocado based acceptance tests Signed-off-by: Peter Maydell <peter.maydell@linaro.org>
This commit is contained in:
Peter Maydell
commit
61fee7f459
|
|
@ -2538,6 +2538,7 @@ F: net/filter-replay.c
|
|||
F: include/sysemu/replay.h
|
||||
F: docs/replay.txt
|
||||
F: stubs/replay.c
|
||||
F: tests/acceptance/replay_kernel.py
|
||||
|
||||
IOVA Tree
|
||||
M: Peter Xu <peterx@redhat.com>
|
||||
|
|
@ -2945,6 +2946,14 @@ S: Maintained
|
|||
F: tests/tcg/Makefile
|
||||
F: tests/tcg/Makefile.include
|
||||
|
||||
Acceptance (Integration) Testing with the Avocado framework
|
||||
W: https://trello.com/b/6Qi1pxVn/avocado-qemu
|
||||
R: Cleber Rosa <crosa@redhat.com>
|
||||
R: Philippe Mathieu-Daudé <philmd@redhat.com>
|
||||
R: Wainer dos Santos Moschetta <wainersm@redhat.com>
|
||||
S: Odd Fixes
|
||||
F: tests/acceptance/
|
||||
|
||||
Documentation
|
||||
-------------
|
||||
Build system architecture
|
||||
|
|
|
|||
|
|
@ -0,0 +1,300 @@
|
|||
# Record/replay test that boots a Linux kernel
|
||||
#
|
||||
# Copyright (c) 2020 ISP RAS
|
||||
#
|
||||
# Author:
|
||||
# Pavel Dovgalyuk <Pavel.Dovgaluk@ispras.ru>
|
||||
#
|
||||
# This work is licensed under the terms of the GNU GPL, version 2 or
|
||||
# later. See the COPYING file in the top-level directory.
|
||||
|
||||
import os
|
||||
import logging
|
||||
import time
|
||||
|
||||
from avocado import skipIf
|
||||
from avocado_qemu import wait_for_console_pattern
|
||||
from avocado.utils import archive
|
||||
from avocado.utils import process
|
||||
from boot_linux_console import LinuxKernelTest
|
||||
|
||||
class ReplayKernel(LinuxKernelTest):
|
||||
"""
|
||||
Boots a Linux kernel in record mode and checks that the console
|
||||
is operational and the kernel command line is properly passed
|
||||
from QEMU to the kernel.
|
||||
Then replays the same scenario and verifies, that QEMU correctly
|
||||
terminates.
|
||||
"""
|
||||
|
||||
timeout = 90
|
||||
KERNEL_COMMON_COMMAND_LINE = 'printk.time=1 panic=-1 '
|
||||
|
||||
def run_vm(self, kernel_path, kernel_command_line, console_pattern,
|
||||
record, shift, args, replay_path):
|
||||
logger = logging.getLogger('replay')
|
||||
start_time = time.time()
|
||||
vm = self.get_vm()
|
||||
vm.set_console()
|
||||
if record:
|
||||
logger.info('recording the execution...')
|
||||
mode = 'record'
|
||||
else:
|
||||
logger.info('replaying the execution...')
|
||||
mode = 'replay'
|
||||
vm.add_args('-icount', 'shift=%s,rr=%s,rrfile=%s' %
|
||||
(shift, mode, replay_path),
|
||||
'-kernel', kernel_path,
|
||||
'-append', kernel_command_line,
|
||||
'-net', 'none',
|
||||
'-no-reboot')
|
||||
if args:
|
||||
vm.add_args(*args)
|
||||
vm.launch()
|
||||
self.wait_for_console_pattern(console_pattern, vm)
|
||||
if record:
|
||||
vm.shutdown()
|
||||
logger.info('finished the recording with log size %s bytes'
|
||||
% os.path.getsize(replay_path))
|
||||
else:
|
||||
vm.wait()
|
||||
logger.info('successfully finished the replay')
|
||||
elapsed = time.time() - start_time
|
||||
logger.info('elapsed time %.2f sec' % elapsed)
|
||||
return elapsed
|
||||
|
||||
def run_rr(self, kernel_path, kernel_command_line, console_pattern,
|
||||
shift=7, args=None):
|
||||
replay_path = os.path.join(self.workdir, 'replay.bin')
|
||||
t1 = self.run_vm(kernel_path, kernel_command_line, console_pattern,
|
||||
True, shift, args, replay_path)
|
||||
t2 = self.run_vm(kernel_path, kernel_command_line, console_pattern,
|
||||
False, shift, args, replay_path)
|
||||
logger = logging.getLogger('replay')
|
||||
logger.info('replay overhead {:.2%}'.format(t2 / t1 - 1))
|
||||
|
||||
@skipIf(os.getenv('CONTINUOUS_INTEGRATION'), 'Running on Travis-CI')
|
||||
def test_x86_64_pc(self):
|
||||
"""
|
||||
:avocado: tags=arch:x86_64
|
||||
:avocado: tags=machine:pc
|
||||
"""
|
||||
kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
|
||||
'/linux/releases/29/Everything/x86_64/os/images/pxeboot'
|
||||
'/vmlinuz')
|
||||
kernel_hash = '23bebd2680757891cf7adedb033532163a792495'
|
||||
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
|
||||
|
||||
kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0'
|
||||
console_pattern = 'VFS: Cannot open root device'
|
||||
|
||||
self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=5)
|
||||
|
||||
def test_aarch64_virt(self):
|
||||
"""
|
||||
:avocado: tags=arch:aarch64
|
||||
:avocado: tags=machine:virt
|
||||
:avocado: tags=cpu:cortex-a53
|
||||
"""
|
||||
kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
|
||||
'/linux/releases/29/Everything/aarch64/os/images/pxeboot'
|
||||
'/vmlinuz')
|
||||
kernel_hash = '8c73e469fc6ea06a58dc83a628fc695b693b8493'
|
||||
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
|
||||
|
||||
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
|
||||
'console=ttyAMA0')
|
||||
console_pattern = 'VFS: Cannot open root device'
|
||||
|
||||
self.run_rr(kernel_path, kernel_command_line, console_pattern,
|
||||
args=('-cpu', 'cortex-a53'))
|
||||
|
||||
def test_arm_virt(self):
|
||||
"""
|
||||
:avocado: tags=arch:arm
|
||||
:avocado: tags=machine:virt
|
||||
"""
|
||||
kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
|
||||
'/linux/releases/29/Everything/armhfp/os/images/pxeboot'
|
||||
'/vmlinuz')
|
||||
kernel_hash = 'e9826d741b4fb04cadba8d4824d1ed3b7fb8b4d4'
|
||||
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
|
||||
|
||||
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
|
||||
'console=ttyAMA0')
|
||||
console_pattern = 'VFS: Cannot open root device'
|
||||
|
||||
self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=1)
|
||||
|
||||
def test_arm_cubieboard_initrd(self):
|
||||
"""
|
||||
:avocado: tags=arch:arm
|
||||
:avocado: tags=machine:cubieboard
|
||||
"""
|
||||
deb_url = ('https://apt.armbian.com/pool/main/l/'
|
||||
'linux-4.20.7-sunxi/linux-image-dev-sunxi_5.75_armhf.deb')
|
||||
deb_hash = '1334c29c44d984ffa05ed10de8c3361f33d78315'
|
||||
deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash)
|
||||
kernel_path = self.extract_from_deb(deb_path,
|
||||
'/boot/vmlinuz-4.20.7-sunxi')
|
||||
dtb_path = '/usr/lib/linux-image-dev-sunxi/sun4i-a10-cubieboard.dtb'
|
||||
dtb_path = self.extract_from_deb(deb_path, dtb_path)
|
||||
initrd_url = ('https://github.com/groeck/linux-build-test/raw/'
|
||||
'2eb0a73b5d5a28df3170c546ddaaa9757e1e0848/rootfs/'
|
||||
'arm/rootfs-armv5.cpio.gz')
|
||||
initrd_hash = '2b50f1873e113523967806f4da2afe385462ff9b'
|
||||
initrd_path_gz = self.fetch_asset(initrd_url, asset_hash=initrd_hash)
|
||||
initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
|
||||
archive.gzip_uncompress(initrd_path_gz, initrd_path)
|
||||
|
||||
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
|
||||
'console=ttyS0,115200 '
|
||||
'usbcore.nousb '
|
||||
'panic=-1 noreboot')
|
||||
console_pattern = 'Boot successful.'
|
||||
self.run_rr(kernel_path, kernel_command_line, console_pattern, shift=1,
|
||||
args=('-dtb', dtb_path,
|
||||
'-initrd', initrd_path,
|
||||
'-no-reboot'))
|
||||
|
||||
def test_ppc64_pseries(self):
|
||||
"""
|
||||
:avocado: tags=arch:ppc64
|
||||
:avocado: tags=machine:pseries
|
||||
"""
|
||||
kernel_url = ('https://archives.fedoraproject.org/pub/archive'
|
||||
'/fedora-secondary/releases/29/Everything/ppc64le/os'
|
||||
'/ppc/ppc64/vmlinuz')
|
||||
kernel_hash = '3fe04abfc852b66653b8c3c897a59a689270bc77'
|
||||
kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
|
||||
|
||||
kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=hvc0'
|
||||
# icount is not good enough for PPC64 for complete boot yet
|
||||
console_pattern = 'Kernel command line: %s' % kernel_command_line
|
||||
self.run_rr(kernel_path, kernel_command_line, console_pattern)
|
||||
|
||||
def test_m68k_q800(self):
|
||||
"""
|
||||
:avocado: tags=arch:m68k
|
||||
:avocado: tags=machine:q800
|
||||
"""
|
||||
deb_url = ('https://snapshot.debian.org/archive/debian-ports'
|
||||
'/20191021T083923Z/pool-m68k/main'
|
||||
'/l/linux/kernel-image-5.3.0-1-m68k-di_5.3.7-1_m68k.udeb')
|
||||
deb_hash = '044954bb9be4160a3ce81f8bc1b5e856b75cccd1'
|
||||
deb_path = self.fetch_asset(deb_url, asset_hash=deb_hash)
|
||||
kernel_path = self.extract_from_deb(deb_path,
|
||||
'/boot/vmlinux-5.3.0-1-m68k')
|
||||
|
||||
kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
|
||||
'console=ttyS0 vga=off')
|
||||
console_pattern = 'No filesystem could mount root'
|
||||
self.run_rr(kernel_path, kernel_command_line, console_pattern)
|
||||
|
||||
def do_test_advcal_2018(self, file_path, kernel_name, args=None):
|
||||
archive.extract(file_path, self.workdir)
|
||||
|
||||
for entry in os.scandir(self.workdir):
|
||||
if entry.name.startswith('day') and entry.is_dir():
|
||||
kernel_path = os.path.join(entry.path, kernel_name)
|
||||
break
|
||||
|
||||
kernel_command_line = ''
|
||||
console_pattern = 'QEMU advent calendar'
|
||||
self.run_rr(kernel_path, kernel_command_line, console_pattern,
|
||||
args=args)
|
||||
|
||||
def test_arm_vexpressa9(self):
|
||||
"""
|
||||
:avocado: tags=arch:arm
|
||||
:avocado: tags=machine:vexpress-a9
|
||||
"""
|
||||
tar_hash = '32b7677ce8b6f1471fb0059865f451169934245b'
|
||||
tar_url = ('https://www.qemu-advent-calendar.org'
|
||||
'/2018/download/day16.tar.xz')
|
||||
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
|
||||
dtb_path = self.workdir + '/day16/vexpress-v2p-ca9.dtb'
|
||||
self.do_test_advcal_2018(file_path, 'winter.zImage',
|
||||
args=('-dtb', dtb_path))
|
||||
|
||||
def test_m68k_mcf5208evb(self):
|
||||
"""
|
||||
:avocado: tags=arch:m68k
|
||||
:avocado: tags=machine:mcf5208evb
|
||||
"""
|
||||
tar_hash = 'ac688fd00561a2b6ce1359f9ff6aa2b98c9a570c'
|
||||
tar_url = ('https://www.qemu-advent-calendar.org'
|
||||
'/2018/download/day07.tar.xz')
|
||||
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
|
||||
self.do_test_advcal_2018(file_path, 'sanity-clause.elf')
|
||||
|
||||
def test_microblaze_s3adsp1800(self):
|
||||
"""
|
||||
:avocado: tags=arch:microblaze
|
||||
:avocado: tags=machine:petalogix-s3adsp1800
|
||||
"""
|
||||
tar_hash = '08bf3e3bfb6b6c7ce1e54ab65d54e189f2caf13f'
|
||||
tar_url = ('https://www.qemu-advent-calendar.org'
|
||||
'/2018/download/day17.tar.xz')
|
||||
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
|
||||
self.do_test_advcal_2018(file_path, 'ballerina.bin')
|
||||
|
||||
def test_ppc64_e500(self):
|
||||
"""
|
||||
:avocado: tags=arch:ppc64
|
||||
:avocado: tags=machine:ppce500
|
||||
:avocado: tags=cpu:e5500
|
||||
"""
|
||||
tar_hash = '6951d86d644b302898da2fd701739c9406527fe1'
|
||||
tar_url = ('https://www.qemu-advent-calendar.org'
|
||||
'/2018/download/day19.tar.xz')
|
||||
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
|
||||
self.do_test_advcal_2018(file_path, 'uImage', ('-cpu', 'e5500'))
|
||||
|
||||
def test_ppc_g3beige(self):
|
||||
"""
|
||||
:avocado: tags=arch:ppc
|
||||
:avocado: tags=machine:g3beige
|
||||
"""
|
||||
tar_hash = 'e0b872a5eb8fdc5bed19bd43ffe863900ebcedfc'
|
||||
tar_url = ('https://www.qemu-advent-calendar.org'
|
||||
'/2018/download/day15.tar.xz')
|
||||
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
|
||||
self.do_test_advcal_2018(file_path, 'invaders.elf',
|
||||
args=('-M', 'graphics=off'))
|
||||
|
||||
def test_ppc_mac99(self):
|
||||
"""
|
||||
:avocado: tags=arch:ppc
|
||||
:avocado: tags=machine:mac99
|
||||
"""
|
||||
tar_hash = 'e0b872a5eb8fdc5bed19bd43ffe863900ebcedfc'
|
||||
tar_url = ('https://www.qemu-advent-calendar.org'
|
||||
'/2018/download/day15.tar.xz')
|
||||
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
|
||||
self.do_test_advcal_2018(file_path, 'invaders.elf',
|
||||
args=('-M', 'graphics=off'))
|
||||
|
||||
def test_sparc_ss20(self):
|
||||
"""
|
||||
:avocado: tags=arch:sparc
|
||||
:avocado: tags=machine:SS-20
|
||||
"""
|
||||
tar_hash = 'b18550d5d61c7615d989a06edace051017726a9f'
|
||||
tar_url = ('https://www.qemu-advent-calendar.org'
|
||||
'/2018/download/day11.tar.xz')
|
||||
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
|
||||
self.do_test_advcal_2018(file_path, 'zImage.elf')
|
||||
|
||||
def test_xtensa_lx60(self):
|
||||
"""
|
||||
:avocado: tags=arch:xtensa
|
||||
:avocado: tags=machine:lx60
|
||||
:avocado: tags=cpu:dc233c
|
||||
"""
|
||||
tar_hash = '49e88d9933742f0164b60839886c9739cb7a0d34'
|
||||
tar_url = ('https://www.qemu-advent-calendar.org'
|
||||
'/2018/download/day02.tar.xz')
|
||||
file_path = self.fetch_asset(tar_url, asset_hash=tar_hash)
|
||||
self.do_test_advcal_2018(file_path, 'santas-sleigh-ride.elf',
|
||||
args=('-cpu', 'dc233c'))
|
||||
Loading…
Reference in New Issue