From bbb0151cf2e82489120a15df5e2eb9653312b0ec Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:34:52 -0400 Subject: [PATCH 01/14] qga: treat get-guest-fsinfo as "best effort" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In some container environments, there may be references to block devices witnessable from a container through /proc/self/mountinfo that reference devices we simply don't have access to in the container, and cannot provide information about. Instead of failing the entire fsinfo command, return stub information for these failed lookups. This allows test-qga to pass under docker tests, which are in turn used by the CentOS VM tests. Signed-off-by: John Snow Reviewed-by: Marc-André Lureau Message-Id: <20220708153503.18864-2-jsnow@redhat.com> Signed-off-by: Thomas Huth --- qga/commands-posix.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/qga/commands-posix.c b/qga/commands-posix.c index f18530d85f..954efed01b 100644 --- a/qga/commands-posix.c +++ b/qga/commands-posix.c @@ -1207,7 +1207,15 @@ static void build_guest_fsinfo_for_device(char const *devpath, syspath = realpath(devpath, NULL); if (!syspath) { - error_setg_errno(errp, errno, "realpath(\"%s\")", devpath); + if (errno != ENOENT) { + error_setg_errno(errp, errno, "realpath(\"%s\")", devpath); + return; + } + + /* ENOENT: This devpath may not exist because of container config */ + if (!fs->name) { + fs->name = g_path_get_basename(devpath); + } return; } From 1ab330eae5bc191ed165adc6937fba20ee767b56 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:34:53 -0400 Subject: [PATCH 02/14] tests/vm: use 'cp' instead of 'ln' for temporary vm images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the initial setup fails, you've permanently altered the state of the downloaded image in an unknowable way. Use 'cp' like our other test setup scripts do. Signed-off-by: John Snow Reviewed-by: Thomas Huth Reviewed-by: Daniel P. Berrangé Reviewed-by: Richard Henderson Message-Id: <20220708153503.18864-3-jsnow@redhat.com> Signed-off-by: Thomas Huth --- tests/vm/centos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vm/centos b/tests/vm/centos index 5c7bc1c1a9..be4f6ff2f1 100755 --- a/tests/vm/centos +++ b/tests/vm/centos @@ -34,7 +34,7 @@ class CentosVM(basevm.BaseVM): def build_image(self, img): cimg = self._download_with_cache("https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2") img_tmp = img + ".tmp" - subprocess.check_call(["ln", "-f", cimg, img_tmp]) + subprocess.check_call(['cp', '-f', cimg, img_tmp]) self.exec_qemu_img("resize", img_tmp, "50G") self.boot(img_tmp, extra_args = ["-cdrom", self.gen_cloud_init_iso()]) self.wait_ssh() From 70457c60fea43f34ad10adc6bb60c73feff049f3 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:34:54 -0400 Subject: [PATCH 03/14] tests/vm: switch CentOS 8 to CentOS 8 Stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The old CentOS image didn't work anymore because it was already EOL at the beginning of 2022. Signed-off-by: John Snow Reviewed-by: Thomas Huth Reviewed-by: Daniel P. Berrangé Reviewed-by: Richard Henderson Message-Id: <20220708153503.18864-4-jsnow@redhat.com> Signed-off-by: Thomas Huth --- tests/vm/centos | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/vm/centos b/tests/vm/centos index be4f6ff2f1..3a527c47b3 100755 --- a/tests/vm/centos +++ b/tests/vm/centos @@ -1,8 +1,8 @@ #!/usr/bin/env python3 # -# CentOS image +# CentOS 8 Stream image # -# Copyright 2018 Red Hat Inc. +# Copyright 2018, 2022 Red Hat Inc. # # Authors: # Fam Zheng @@ -32,7 +32,7 @@ class CentosVM(basevm.BaseVM): """ def build_image(self, img): - cimg = self._download_with_cache("https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2") + cimg = self._download_with_cache("https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20220125.1.x86_64.qcow2") img_tmp = img + ".tmp" subprocess.check_call(['cp', '-f', cimg, img_tmp]) self.exec_qemu_img("resize", img_tmp, "50G") From 5d8e7da891f60b0e46ff66a6fad3703d4ce2ba63 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:34:55 -0400 Subject: [PATCH 04/14] tests/vm: switch centos.aarch64 to CentOS 8 Stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch this test over to using a cloud image like the base CentOS8 VM test, which helps make this script a bit simpler too. Note: At time of writing, this test seems pretty flaky when run without KVM support for aarch64. Certain unit tests like migration-test, virtio-net-failover, test-hmp and qom-test seem quite prone to fail under TCG. Still, this is an improvement in that at least pure build tests are functional. Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé Reviewed-by: Richard Henderson Message-Id: <20220708153503.18864-5-jsnow@redhat.com> Signed-off-by: Thomas Huth --- tests/vm/centos.aarch64 | 174 ++++++---------------------------------- 1 file changed, 24 insertions(+), 150 deletions(-) diff --git a/tests/vm/centos.aarch64 b/tests/vm/centos.aarch64 index 96c450f8be..2de7ef6992 100755 --- a/tests/vm/centos.aarch64 +++ b/tests/vm/centos.aarch64 @@ -20,150 +20,38 @@ import time import traceback import aarch64vm + DEFAULT_CONFIG = { 'cpu' : "max", 'machine' : "virt,gic-version=max", - 'install_cmds' : "yum install -y make ninja-build git python3 gcc gcc-c++ flex bison, "\ - "yum install -y glib2-devel perl pixman-devel zlib-devel, "\ - "alternatives --set python /usr/bin/python3, "\ - "sudo dnf config-manager "\ - "--add-repo=https://download.docker.com/linux/centos/docker-ce.repo,"\ - "sudo dnf install --nobest -y docker-ce.aarch64,"\ - "systemctl enable docker", + 'install_cmds' : ( + "dnf config-manager --set-enabled powertools, " + "dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo, " + "dnf install -y make ninja-build git python38 gcc gcc-c++ flex bison "\ + "glib2-devel perl pixman-devel zlib-devel docker-ce.aarch64, " + "systemctl enable docker, " + ), # We increase beyond the default time since during boot # it can take some time (many seconds) to log into the VM. 'ssh_timeout' : 60, } + class CentosAarch64VM(basevm.BaseVM): - name = "centos.aarch64" + name = "centos8.aarch64" arch = "aarch64" - login_prompt = "localhost login:" - prompt = '[root@localhost ~]#' - image_name = "CentOS-8-aarch64-1905-dvd1.iso" - image_link = "http://mirrors.usc.edu/pub/linux/distributions/centos/8.0.1905/isos/aarch64/" + image_name = "CentOS-Stream-GenericCloud-8-20220125.1.aarch64.qcow2" + image_link = "https://cloud.centos.org/centos/8-stream/aarch64/images/" image_link += image_name BUILD_SCRIPT = """ set -e; cd $(mktemp -d); - sudo chmod a+r /dev/vdb; - tar --checkpoint=.10 -xf /dev/vdb; + export SRC_ARCHIVE=/dev/vdb; + sudo chmod a+r $SRC_ARCHIVE; + tar -xf $SRC_ARCHIVE; ./configure {configure_opts}; make --output-sync {target} -j{jobs} {verbose}; """ - def set_key_perm(self): - """Set permissions properly on certain files to allow - ssh access.""" - self.console_wait_send(self.prompt, - "/usr/sbin/restorecon -R -v /root/.ssh\n") - self.console_wait_send(self.prompt, - "/usr/sbin/restorecon -R -v "\ - "/home/{}/.ssh\n".format(self._config["guest_user"])) - - def create_kickstart(self): - """Generate the kickstart file used to generate the centos image.""" - # Start with the template for the kickstart. - ks_file = self._source_path + "/tests/vm/centos-8-aarch64.ks" - subprocess.check_call("cp {} ./ks.cfg".format(ks_file), shell=True) - # Append the ssh keys to the kickstart file - # as the post processing phase of installation. - with open("ks.cfg", "a") as f: - # Add in the root pw and guest user. - rootpw = "rootpw --plaintext {}\n" - f.write(rootpw.format(self._config["root_pass"])) - add_user = "user --groups=wheel --name={} "\ - "--password={} --plaintext\n" - f.write(add_user.format(self._config["guest_user"], - self._config["guest_pass"])) - # Add the ssh keys. - f.write("%post --log=/root/ks-post.log\n") - f.write("mkdir -p /root/.ssh\n") - addkey = 'echo "{}" >> /root/.ssh/authorized_keys\n' - addkey_cmd = addkey.format(self._config["ssh_pub_key"]) - f.write(addkey_cmd) - f.write('mkdir -p /home/{}/.ssh\n'.format(self._config["guest_user"])) - addkey = 'echo "{}" >> /home/{}/.ssh/authorized_keys\n' - addkey_cmd = addkey.format(self._config["ssh_pub_key"], - self._config["guest_user"]) - f.write(addkey_cmd) - f.write("%end\n") - # Take our kickstart file and create an .iso from it. - # The .iso will be provided to qemu as we boot - # from the install dvd. - # Anaconda will recognize the label "OEMDRV" and will - # start the automated installation. - gen_iso_img = 'genisoimage -output ks.iso -volid "OEMDRV" ks.cfg' - subprocess.check_call(gen_iso_img, shell=True) - - def wait_for_shutdown(self): - """We wait for qemu to shutdown the VM and exit. - While this happens we display the console view - for easier debugging.""" - # The image creation is essentially done, - # so whether or not the wait is successful we want to - # wait for qemu to exit (the self.wait()) before we return. - try: - self.console_wait("reboot: Power down") - except Exception as e: - sys.stderr.write("Exception hit\n") - if isinstance(e, SystemExit) and e.code == 0: - return 0 - traceback.print_exc() - finally: - self.wait() - - def build_base_image(self, dest_img): - """Run through the centos installer to create - a base image with name dest_img.""" - # We create the temp image, and only rename - # to destination when we are done. - img = dest_img + ".tmp" - # Create an empty image. - # We will provide this as the install destination. - qemu_img_create = "qemu-img create {} 50G".format(img) - subprocess.check_call(qemu_img_create, shell=True) - - # Create our kickstart file to be fed to the installer. - self.create_kickstart() - # Boot the install dvd with the params as our ks.iso - os_img = self._download_with_cache(self.image_link) - dvd_iso = "centos-8-dvd.iso" - subprocess.check_call(["cp", "-f", os_img, dvd_iso]) - extra_args = "-cdrom ks.iso" - extra_args += " -drive file={},if=none,id=drive1,cache=writeback" - extra_args += " -device virtio-blk,drive=drive1,bootindex=1" - extra_args = extra_args.format(dvd_iso).split(" ") - self.boot(img, extra_args=extra_args) - self.console_wait_send("change the selection", "\n") - # We seem to need to hit esc (chr(27)) twice to abort the - # media check, which takes a long time. - # Waiting a bit seems to be more reliable before hitting esc. - self.console_wait("Checking") - time.sleep(5) - self.console_wait_send("Checking", chr(27)) - time.sleep(5) - self.console_wait_send("Checking", chr(27)) - print("Found Checking") - # Give sufficient time for the installer to create the image. - self.console_init(timeout=7200) - self.wait_for_shutdown() - os.rename(img, dest_img) - print("Done with base image build: {}".format(dest_img)) - - def check_create_base_img(self, img_base, img_dest): - """Create a base image using the installer. - We will use the base image if it exists. - This helps cut down on install time in case we - need to restart image creation, - since the base image creation can take a long time.""" - if not os.path.exists(img_base): - print("Generate new base image: {}".format(img_base)) - self.build_base_image(img_base); - else: - print("Use existing base image: {}".format(img_base)) - # Save a copy of the base image and copy it to dest. - # which we will use going forward. - subprocess.check_call(["cp", img_base, img_dest]) def boot(self, img, extra_args=None): aarch64vm.create_flash_images(self._tmpdir, self._efi_aarch64) @@ -185,42 +73,28 @@ class CentosAarch64VM(basevm.BaseVM): super(CentosAarch64VM, self).boot(img, extra_args=extra_args) def build_image(self, img): + cimg = self._download_with_cache(self.image_link) img_tmp = img + ".tmp" - self.check_create_base_img(img + ".base", img_tmp) - - # Boot the new image for the first time to finish installation. - self.boot(img_tmp) - self.console_init() - self.console_wait_send(self.login_prompt, "root\n") - self.console_wait_send("Password:", - "{}\n".format(self._config["root_pass"])) - - self.set_key_perm() - self.console_wait_send(self.prompt, "rpm -q centos-release\n") - enable_adapter = "sed -i 's/ONBOOT=no/ONBOOT=yes/g'" \ - " /etc/sysconfig/network-scripts/ifcfg-enp0s1\n" - self.console_wait_send(self.prompt, enable_adapter) - self.console_wait_send(self.prompt, "ifup enp0s1\n") - self.console_wait_send(self.prompt, - 'echo "qemu ALL=(ALL) NOPASSWD:ALL" | '\ - 'sudo tee /etc/sudoers.d/qemu\n') - self.console_wait(self.prompt) - - # Rest of the commands we issue through ssh. + subprocess.run(['cp', '-f', cimg, img_tmp]) + self.exec_qemu_img("resize", img_tmp, "50G") + self.boot(img_tmp, extra_args = ["-cdrom", self.gen_cloud_init_iso()]) self.wait_ssh(wait_root=True) + self.ssh_root_check("touch /etc/cloud/cloud-init.disabled") # If the user chooses *not* to do the second phase, # then we will jump right to the graceful shutdown if self._config['install_cmds'] != "": install_cmds = self._config['install_cmds'].split(',') for cmd in install_cmds: - self.ssh_root(cmd) + self.ssh_root_check(cmd) + self.ssh_root("poweroff") - self.wait_for_shutdown() + self.wait() os.rename(img_tmp, img) print("image creation complete: {}".format(img)) return 0 + if __name__ == "__main__": defaults = aarch64vm.get_config_defaults(CentosAarch64VM, DEFAULT_CONFIG) sys.exit(basevm.main(CentosAarch64VM, defaults)) From 47f71f8912dadb6b1bc0d3015eb18253092bcc70 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:34:56 -0400 Subject: [PATCH 05/14] tests/vm: upgrade Ubuntu 18.04 VM to 20.04 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 18.04 has fallen out of our support window, so move ubuntu.aarch64 forward to ubuntu 20.04, which is now our oldest supported Ubuntu release. Notes: This checksum changes periodically; use a fixed point image with a known checksum so that the image isn't re-downloaded on every single invocation. (The checksum for the 18.04 image was already incorrect at the time of writing.) Just like the centos.aarch64 test, this test currently seems very flaky when run as a TCG test. Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé Reviewed-by: Richard Henderson Message-Id: <20220708153503.18864-6-jsnow@redhat.com> Signed-off-by: Thomas Huth --- tests/vm/ubuntu.aarch64 | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/vm/ubuntu.aarch64 b/tests/vm/ubuntu.aarch64 index b291945a7e..666947393b 100755 --- a/tests/vm/ubuntu.aarch64 +++ b/tests/vm/ubuntu.aarch64 @@ -32,9 +32,13 @@ DEFAULT_CONFIG = { class UbuntuAarch64VM(ubuntuvm.UbuntuVM): name = "ubuntu.aarch64" arch = "aarch64" - image_name = "ubuntu-18.04-server-cloudimg-arm64.img" - image_link = "https://cloud-images.ubuntu.com/releases/18.04/release/" + image_name - image_sha256="0fdcba761965735a8a903d8b88df8e47f156f48715c00508e4315c506d7d3cb1" + # NOTE: The Ubuntu 20.04 cloud images are periodically updated. The + # fixed image chosen below is the latest release at time of + # writing. Using a rolling latest instead would mean that the SHA + # would be incorrect at an indeterminate point in the future. + image_name = "focal-server-cloudimg-arm64.img" + image_link = "https://cloud-images.ubuntu.com/focal/20220615/" + image_name + image_sha256="95a027336e197debe88c92ff2e554598e23c409139e1e750b71b3b820b514832" BUILD_SCRIPT = """ set -e; cd $(mktemp -d); From 5e658729b6e6f79cf3afac046f067459f4fcecb2 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:34:57 -0400 Subject: [PATCH 06/14] tests/vm: remove ubuntu.i386 VM test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ubuntu 18.04 is out of our support window, and Ubuntu 20.04 does not support i386 anymore. The debian project does, but they do not provide any cloud images for it, a new expect-style script would have to be written. Since we have i386 cross-compiler tests hosted on GitLab CI, we don't need to support this VM test anymore. Signed-off-by: John Snow Reviewed-by: Thomas Huth Reviewed-by: Daniel P. Berrangé Reviewed-by: Richard Henderson Message-Id: <20220708153503.18864-7-jsnow@redhat.com> Signed-off-by: Thomas Huth --- tests/vm/Makefile.include | 3 +-- tests/vm/ubuntu.i386 | 40 --------------------------------------- 2 files changed, 1 insertion(+), 42 deletions(-) delete mode 100755 tests/vm/ubuntu.i386 diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index 5f5b1fbfe6..a94f0ebf7f 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -17,7 +17,7 @@ EFI_AARCH64 = $(wildcard $(BUILD_DIR)/pc-bios/edk2-aarch64-code.fd) X86_IMAGES := freebsd netbsd openbsd centos fedora haiku.x86_64 ifneq ($(GENISOIMAGE),) -X86_IMAGES += ubuntu.i386 centos +X86_IMAGES += centos ifneq ($(EFI_AARCH64),) ARM64_IMAGES += ubuntu.aarch64 centos.aarch64 endif @@ -48,7 +48,6 @@ vm-help vm-test: @echo " vm-build-fedora - Build QEMU in Fedora VM" ifneq ($(GENISOIMAGE),) @echo " vm-build-centos - Build QEMU in CentOS VM, with Docker" - @echo " vm-build-ubuntu.i386 - Build QEMU in ubuntu i386 VM" ifneq ($(EFI_AARCH64),) @echo " vm-build-ubuntu.aarch64 - Build QEMU in ubuntu aarch64 VM" @echo " vm-build-centos.aarch64 - Build QEMU in CentOS aarch64 VM" diff --git a/tests/vm/ubuntu.i386 b/tests/vm/ubuntu.i386 deleted file mode 100755 index 47681b6f87..0000000000 --- a/tests/vm/ubuntu.i386 +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -# -# Ubuntu i386 image -# -# Copyright 2017 Red Hat Inc. -# -# Authors: -# Fam Zheng -# -# This code is licensed under the GPL version 2 or later. See -# the COPYING file in the top-level directory. -# - -import sys -import basevm -import ubuntuvm - -DEFAULT_CONFIG = { - 'install_cmds' : "apt-get update,"\ - "apt-get build-dep -y qemu,"\ - "apt-get install -y libfdt-dev language-pack-en ninja-build", -} - -class UbuntuX86VM(ubuntuvm.UbuntuVM): - name = "ubuntu.i386" - arch = "i386" - image_link="https://cloud-images.ubuntu.com/releases/bionic/"\ - "release-20191114/ubuntu-18.04-server-cloudimg-i386.img" - image_sha256="28969840626d1ea80bb249c08eef1a4533e8904aa51a327b40f37ac4b4ff04ef" - BUILD_SCRIPT = """ - set -e; - cd $(mktemp -d); - sudo chmod a+r /dev/vdb; - tar -xf /dev/vdb; - ./configure {configure_opts}; - make --output-sync {target} -j{jobs} {verbose}; - """ - -if __name__ == "__main__": - sys.exit(basevm.main(UbuntuX86VM, DEFAULT_CONFIG)) From b967bf134581214280540b2aec1161cee6543f83 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:34:58 -0400 Subject: [PATCH 07/14] tests/vm: remove duplicate 'centos' VM test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is listed twice by accident; we require genisoimage to run the test, so remove the unconditional entry. Signed-off-by: John Snow Reviewed-by: Thomas Huth Reviewed-by: Daniel P. Berrangé Reviewed-by: Richard Henderson Message-Id: <20220708153503.18864-8-jsnow@redhat.com> Signed-off-by: Thomas Huth --- tests/vm/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index a94f0ebf7f..8d2a164552 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -15,7 +15,7 @@ endif EFI_AARCH64 = $(wildcard $(BUILD_DIR)/pc-bios/edk2-aarch64-code.fd) -X86_IMAGES := freebsd netbsd openbsd centos fedora haiku.x86_64 +X86_IMAGES := freebsd netbsd openbsd fedora haiku.x86_64 ifneq ($(GENISOIMAGE),) X86_IMAGES += centos ifneq ($(EFI_AARCH64),) From eaf46a65ab41b120a07204bedccadf1844c73009 Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:34:59 -0400 Subject: [PATCH 08/14] tests/vm: add 1GB extra memory per core MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If you try to run a 16 or 32 threaded test, you're going to run out of memory very quickly with qom-test and a few others. Bump the memory limit to try to scale with larger-core machines. Granted, this means that a 16 core processor is going to ask for 16GB, but you *probably* meet that requirement if you have such a machine. 512MB per core didn't seem to be enough to avoid ENOMEM and SIGABRTs in the test cases in practice on a six core machine; so I bumped it up to 1GB which seemed to help. Add this magic in early to the configuration process so that the config file, if provided, can still override it. Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé Acked-by: Richard Henderson Message-Id: <20220708153503.18864-9-jsnow@redhat.com> Signed-off-by: Thomas Huth --- tests/vm/basevm.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index d7d0413df3..4fd9af10b7 100644 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -99,6 +99,11 @@ class BaseVM(object): self._source_path = args.source_path # Allow input config to override defaults. self._config = DEFAULT_CONFIG.copy() + + # 1GB per core, minimum of 4. This is only a default. + mem = max(4, args.jobs) + self._config['memory'] = f"{mem}G" + if config != None: self._config.update(config) self.validate_ssh_keys() From 28a48ed5f7bdbddb6955f732402ca9f51e56b23a Mon Sep 17 00:00:00 2001 From: John Snow Date: Fri, 8 Jul 2022 11:35:00 -0400 Subject: [PATCH 09/14] tests/vm: Remove docker cross-compile test from CentOS VM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fedora container has since been split apart, so there's no suitable nearby target that would support "test-mingw" as it requires both x32 and x64 support -- so either fedora-cross-win32 nor fedora-cross-win64 would be truly suitable. Just remove this test as superfluous with our current CI infrastructure. Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé Reviewed-by: Richard Henderson Message-Id: <20220708153503.18864-10-jsnow@redhat.com> Signed-off-by: Thomas Huth --- tests/vm/centos | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/vm/centos b/tests/vm/centos index 3a527c47b3..097a9ca14d 100755 --- a/tests/vm/centos +++ b/tests/vm/centos @@ -28,7 +28,6 @@ class CentosVM(basevm.BaseVM): tar -xf $SRC_ARCHIVE; make docker-test-block@centos8 {verbose} J={jobs} NETWORK=1; make docker-test-quick@centos8 {verbose} J={jobs} NETWORK=1; - make docker-test-mingw@fedora {verbose} J={jobs} NETWORK=1; """ def build_image(self, img): From e18f27d9ed6132b774910e6ce6999cfc59822e67 Mon Sep 17 00:00:00 2001 From: Song Gao Date: Wed, 13 Jul 2022 10:02:58 +0800 Subject: [PATCH 10/14] qtest/machine-none: Add LoongArch support Update the cpu_maps[] to support the LoongArch target. Signed-off-by: Song Gao Reviewed-by: Richard Henderson Message-Id: <20220713020258.601424-1-gaosong@loongson.cn> Signed-off-by: Thomas Huth --- tests/qtest/machine-none-test.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/qtest/machine-none-test.c b/tests/qtest/machine-none-test.c index d0f8cd9902..f92fab479f 100644 --- a/tests/qtest/machine-none-test.c +++ b/tests/qtest/machine-none-test.c @@ -54,6 +54,7 @@ static struct arch2cpu cpus_map[] = { { "riscv64", "rv64" }, { "riscv32", "rv32" }, { "rx", "rx62n" }, + { "loongarch64", "la464"}, }; static const char *get_cpu_model_by_arch(const char *arch) From c4f8ce24de81162d925ca24db8adb194b47fffb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Fri, 3 Sep 2021 19:45:05 +0200 Subject: [PATCH 11/14] tests/unit: Replace g_memdup() by g_memdup2() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per https://discourse.gnome.org/t/port-your-module-from-g-memdup-to-g-memdup2-now/5538 The old API took the size of the memory to duplicate as a guint, whereas most memory functions take memory sizes as a gsize. This made it easy to accidentally pass a gsize to g_memdup(). For large values, that would lead to a silent truncation of the size from 64 to 32 bits, and result in a heap area being returned which is significantly smaller than what the caller expects. This can likely be exploited in various modules to cause a heap buffer overflow. Replace g_memdup() by the safer g_memdup2() wrapper. Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20210903174510.751630-24-philmd@redhat.com> Signed-off-by: Thomas Huth --- tests/unit/ptimer-test.c | 22 +++++++++++----------- tests/unit/test-iov.c | 26 +++++++++++++------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/unit/ptimer-test.c b/tests/unit/ptimer-test.c index a80ef5aff4..04b5f4e3d0 100644 --- a/tests/unit/ptimer-test.c +++ b/tests/unit/ptimer-test.c @@ -798,64 +798,64 @@ static void add_ptimer_tests(uint8_t policy) g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/set_count policy=%s", policy_name), - g_memdup(&policy, 1), check_set_count, g_free); + g_memdup2(&policy, 1), check_set_count, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/set_limit policy=%s", policy_name), - g_memdup(&policy, 1), check_set_limit, g_free); + g_memdup2(&policy, 1), check_set_limit, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/oneshot policy=%s", policy_name), - g_memdup(&policy, 1), check_oneshot, g_free); + g_memdup2(&policy, 1), check_oneshot, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/periodic policy=%s", policy_name), - g_memdup(&policy, 1), check_periodic, g_free); + g_memdup2(&policy, 1), check_periodic, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/on_the_fly_mode_change policy=%s", policy_name), - g_memdup(&policy, 1), check_on_the_fly_mode_change, g_free); + g_memdup2(&policy, 1), check_on_the_fly_mode_change, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/on_the_fly_period_change policy=%s", policy_name), - g_memdup(&policy, 1), check_on_the_fly_period_change, g_free); + g_memdup2(&policy, 1), check_on_the_fly_period_change, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/on_the_fly_freq_change policy=%s", policy_name), - g_memdup(&policy, 1), check_on_the_fly_freq_change, g_free); + g_memdup2(&policy, 1), check_on_the_fly_freq_change, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/run_with_period_0 policy=%s", policy_name), - g_memdup(&policy, 1), check_run_with_period_0, g_free); + g_memdup2(&policy, 1), check_run_with_period_0, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/run_with_delta_0 policy=%s", policy_name), - g_memdup(&policy, 1), check_run_with_delta_0, g_free); + g_memdup2(&policy, 1), check_run_with_delta_0, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/periodic_with_load_0 policy=%s", policy_name), - g_memdup(&policy, 1), check_periodic_with_load_0, g_free); + g_memdup2(&policy, 1), check_periodic_with_load_0, g_free); g_free(tmp); g_test_add_data_func_full( tmp = g_strdup_printf("/ptimer/oneshot_with_load_0 policy=%s", policy_name), - g_memdup(&policy, 1), check_oneshot_with_load_0, g_free); + g_memdup2(&policy, 1), check_oneshot_with_load_0, g_free); g_free(tmp); } diff --git a/tests/unit/test-iov.c b/tests/unit/test-iov.c index 93bda00f0e..6f7623d310 100644 --- a/tests/unit/test-iov.c +++ b/tests/unit/test-iov.c @@ -172,7 +172,7 @@ static void test_io(void) } iov_from_buf(iov, niov, 0, buf, sz); - siov = g_memdup(iov, sizeof(*iov) * niov); + siov = g_memdup2(iov, sizeof(*iov) * niov); if (socketpair(PF_UNIX, SOCK_STREAM, 0, sv) < 0) { perror("socketpair"); @@ -349,7 +349,7 @@ static void test_discard_front_undo(void) /* Discard zero bytes */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_tmp = iov; iov_cnt_tmp = iov_cnt; iov_discard_front_undoable(&iov_tmp, &iov_cnt_tmp, 0, &undo); @@ -360,7 +360,7 @@ static void test_discard_front_undo(void) /* Discard more bytes than vector size */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_tmp = iov; iov_cnt_tmp = iov_cnt; size = iov_size(iov, iov_cnt); @@ -372,7 +372,7 @@ static void test_discard_front_undo(void) /* Discard entire vector */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_tmp = iov; iov_cnt_tmp = iov_cnt; size = iov_size(iov, iov_cnt); @@ -384,7 +384,7 @@ static void test_discard_front_undo(void) /* Discard within first element */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_tmp = iov; iov_cnt_tmp = iov_cnt; size = g_test_rand_int_range(1, iov->iov_len); @@ -396,7 +396,7 @@ static void test_discard_front_undo(void) /* Discard entire first element */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_tmp = iov; iov_cnt_tmp = iov_cnt; iov_discard_front_undoable(&iov_tmp, &iov_cnt_tmp, iov->iov_len, &undo); @@ -407,7 +407,7 @@ static void test_discard_front_undo(void) /* Discard within second element */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_tmp = iov; iov_cnt_tmp = iov_cnt; size = iov->iov_len + g_test_rand_int_range(1, iov[1].iov_len); @@ -498,7 +498,7 @@ static void test_discard_back_undo(void) /* Discard zero bytes */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_cnt_tmp = iov_cnt; iov_discard_back_undoable(iov, &iov_cnt_tmp, 0, &undo); iov_discard_undo(&undo); @@ -508,7 +508,7 @@ static void test_discard_back_undo(void) /* Discard more bytes than vector size */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_cnt_tmp = iov_cnt; size = iov_size(iov, iov_cnt); iov_discard_back_undoable(iov, &iov_cnt_tmp, size + 1, &undo); @@ -519,7 +519,7 @@ static void test_discard_back_undo(void) /* Discard entire vector */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_cnt_tmp = iov_cnt; size = iov_size(iov, iov_cnt); iov_discard_back_undoable(iov, &iov_cnt_tmp, size, &undo); @@ -530,7 +530,7 @@ static void test_discard_back_undo(void) /* Discard within last element */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_cnt_tmp = iov_cnt; size = g_test_rand_int_range(1, iov[iov_cnt - 1].iov_len); iov_discard_back_undoable(iov, &iov_cnt_tmp, size, &undo); @@ -541,7 +541,7 @@ static void test_discard_back_undo(void) /* Discard entire last element */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_cnt_tmp = iov_cnt; size = iov[iov_cnt - 1].iov_len; iov_discard_back_undoable(iov, &iov_cnt_tmp, size, &undo); @@ -552,7 +552,7 @@ static void test_discard_back_undo(void) /* Discard within second-to-last element */ iov_random(&iov, &iov_cnt); - iov_orig = g_memdup(iov, sizeof(iov[0]) * iov_cnt); + iov_orig = g_memdup2(iov, sizeof(iov[0]) * iov_cnt); iov_cnt_tmp = iov_cnt; size = iov[iov_cnt - 1].iov_len + g_test_rand_int_range(1, iov[iov_cnt - 2].iov_len); From 2d2e4843b641f53c8016978f73675da7b66136fb Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Mon, 11 Jul 2022 11:53:00 +0200 Subject: [PATCH 12/14] Replace 'whitelist' with 'allow' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's use more inclusive language here and avoid terms that are frowned upon nowadays. Message-Id: <20220711095300.60462-1-thuth@redhat.com> Reviewed-by: John Snow Reviewed-by: Alex Bennée Signed-off-by: Thomas Huth --- docs/devel/submitting-a-patch.rst | 2 +- docs/tools/qemu-nbd.rst | 2 +- scripts/vmstate-static-checker.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/devel/submitting-a-patch.rst b/docs/devel/submitting-a-patch.rst index 09a8d12c2c..fec33ce148 100644 --- a/docs/devel/submitting-a-patch.rst +++ b/docs/devel/submitting-a-patch.rst @@ -39,7 +39,7 @@ ideas from other posts. If you do subscribe, be prepared for a high volume of email, often over one thousand messages in a week. The list is moderated; first-time posts from an email address (whether or not you subscribed) may be subject to some delay while waiting for a moderator -to whitelist your address. +to allow your address. The larger your contribution is, or if you plan on becoming a long-term contributor, then the more important the rest of this page becomes. diff --git a/docs/tools/qemu-nbd.rst b/docs/tools/qemu-nbd.rst index 8e08a29e89..faf6349ea5 100644 --- a/docs/tools/qemu-nbd.rst +++ b/docs/tools/qemu-nbd.rst @@ -225,7 +225,7 @@ disconnects: qemu-nbd -f qcow2 file.qcow2 Start a long-running server listening with encryption on port 10810, -and whitelist clients with a specific X.509 certificate to connect to +and allow clients with a specific X.509 certificate to connect to a 1 megabyte subset of a raw file, using the export name 'subset': :: diff --git a/scripts/vmstate-static-checker.py b/scripts/vmstate-static-checker.py index 539ead62b4..b369388360 100755 --- a/scripts/vmstate-static-checker.py +++ b/scripts/vmstate-static-checker.py @@ -40,7 +40,7 @@ def check_fields_match(name, s_field, d_field): return True # Some fields changed names between qemu versions. This list - # is used to whitelist such changes in each section / description. + # is used to allow such changes in each section / description. changed_names = { 'apic': ['timer', 'timer_expiry'], 'e1000': ['dev', 'parent_obj'], From 0a979a1320a4ea10c6a26c0d133a147fee5b5ecf Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Mon, 18 Jul 2022 18:42:11 +0200 Subject: [PATCH 13/14] util: Fix broken build on Haiku MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A recent commit moved some Haiku-specific code parts from oslib-posix.c to cutils.c, but failed to move the corresponding header #include statement, too, so "make vm-build-haiku.x86_64" is currently broken. Fix it by moving the header #include, too. Fixes: 06680b15b4 ("include: move qemu_*_exec_dir() to cutils") Message-Id: <20220718172026.139004-1-thuth@redhat.com> Reviewed-by: Marc-André Lureau Signed-off-by: Thomas Huth --- util/cutils.c | 4 ++++ util/oslib-posix.c | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/util/cutils.c b/util/cutils.c index 8199dac598..cb43dda213 100644 --- a/util/cutils.c +++ b/util/cutils.c @@ -35,6 +35,10 @@ #include #endif +#ifdef __HAIKU__ +#include +#endif + #ifdef G_OS_WIN32 #include #include diff --git a/util/oslib-posix.c b/util/oslib-posix.c index 7a34c1657c..bffec18869 100644 --- a/util/oslib-posix.c +++ b/util/oslib-posix.c @@ -62,10 +62,6 @@ #include #endif -#ifdef __HAIKU__ -#include -#endif - #include "qemu/mmap-alloc.h" #ifdef CONFIG_DEBUG_STACK_USAGE From 9b0ecfaba5920ddf8601d7b31746a428aa3779c6 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Mon, 11 Jul 2022 11:57:21 +0200 Subject: [PATCH 14/14] python/qemu/qmp/legacy: Replace 'returns-whitelist' with the correct type 'returns-whitelist' has been renamed to 'command-returns-exceptions' in commit b86df3747848 ("qapi: Rename pragma *-whitelist to *-exceptions"). Message-Id: <20220711095721.61280-1-thuth@redhat.com> Reviewed-by: John Snow Signed-off-by: Thomas Huth --- python/qemu/qmp/legacy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/qemu/qmp/legacy.py b/python/qemu/qmp/legacy.py index 03b5574618..1951754455 100644 --- a/python/qemu/qmp/legacy.py +++ b/python/qemu/qmp/legacy.py @@ -50,7 +50,7 @@ QMPObject = Dict[str, object] # QMPMessage can be outgoing commands or incoming events/returns. # QMPReturnValue is usually a dict/json object, but due to QAPI's -# 'returns-whitelist', it can actually be anything. +# 'command-returns-exceptions', it can actually be anything. # # {'return': {}} is a QMPMessage, # {} is the QMPReturnValue.