From 6a1eef06277c0e187a09a847bc3f90ae86e892c5 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Mon, 6 Apr 2020 12:56:00 +0200 Subject: [PATCH 01/17] QEMU config disk - initial implementation. Ref #2958 (cherry picked from commit b69965791df773f75cbca76f74c8931afeae2ff0) --- gns3server/compute/qemu/qemu_vm.py | 152 +++++++++++++++++++++++++---- 1 file changed, 133 insertions(+), 19 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 69494b51..1eeeb8c0 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -38,6 +38,7 @@ from gns3server.utils.asyncio import subprocess_check_output, cancellable_wait_r from .qemu_error import QemuError from .utils.qcow2 import Qcow2, Qcow2Error from ..adapters.ethernet_adapter import EthernetAdapter +from ..error import NodeError, ImageMissingError from ..nios.nio_udp import NIOUDP from ..nios.nio_tap import NIOTAP from ..base_node import BaseNode @@ -125,6 +126,22 @@ class QemuVM(BaseNode): self.mac_address = "" # this will generate a MAC address self.adapters = 1 # creates 1 adapter by default + + # config disk + self.config_disk_name = "config.img" + if not shutil.which("mcopy"): + log.warning("Config disk: 'mtools' are not installed.") + self.config_disk_name = "" + self.config_disk_image = "" + else: + try: + self.config_disk_image = self.manager.get_abs_image_path( + self.config_disk_name, self.project.path) + except (NodeError, ImageMissingError) as e: + log.warning("Config disk: {}".format(e)) + self.config_disk_name = "" + self.config_disk_image = "" + log.info('QEMU VM "{name}" [{id}] has been created'.format(name=self._name, id=self._id)) @property @@ -1115,6 +1132,7 @@ class QemuVM(BaseNode): self._stop_cpulimit() if self.on_close != "save_vm_state": await self._clear_save_vm_stated() + await self._export_config() await super().stop() async def _open_qemu_monitor_connection_vm(self, timeout=10): @@ -1627,6 +1645,96 @@ class QemuVM(BaseNode): log.info("{} returned with {}".format(self._get_qemu_img(), retcode)) return retcode + async def _mcopy(self, *args): + env = os.environ + env["MTOOLSRC"] = 'mtoolsrc' + try: + process = await asyncio.create_subprocess_exec("mcopy", *args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.working_dir, env=env) + (stdout, _) = await process.communicate() + retcode = process.returncode + except (OSError, subprocess.SubprocessError) as e: + log.error("mcopy failure: {}".format(e)) + return 1 + if retcode != 0: + stdout = stdout.decode("utf-8").rstrip() + if stdout: + log.error("mcopy failure: {}".format(stdout)) + else: + log.error("mcopy failure: return code {}".format(retcode)) + return retcode + + async def _export_config(self): + disk_name = getattr(self, "config_disk_name") + if not disk_name or \ + not os.path.exists(os.path.join(self.working_dir, disk_name)): + return + config_dir = os.path.join(self.working_dir, "configs") + zip_file = os.path.join(self.working_dir, "config.zip") + try: + shutil.rmtree(config_dir, ignore_errors=True) + os.mkdir(config_dir) + if os.path.exists(zip_file): + os.remove(zip_file) + if await self._mcopy("-s", "-m", "x:/", config_dir) == 0: + shutil.make_archive(zip_file[:-4], "zip", config_dir) + except OSError as e: + log.error("Can't export config: {}".format(e)) + finally: + shutil.rmtree(config_dir, ignore_errors=True) + + async def _import_config(self): + disk_name = getattr(self, "config_disk_name") + zip_file = os.path.join(self.working_dir, "config.zip") + if not disk_name or not os.path.exists(zip_file): + return + config_dir = os.path.join(self.working_dir, "configs") + disk = os.path.join(self.working_dir, disk_name) + try: + shutil.rmtree(config_dir, ignore_errors=True) + os.mkdir(config_dir) + shutil.unpack_archive(zip_file, config_dir) + shutil.copyfile(getattr(self, "config_disk_image"), disk) + config_files = [os.path.join(config_dir, fname) + for fname in os.listdir(config_dir)] + if config_files: + if await self._mcopy("-s", "-m", *config_files, "x:/") != 0: + os.remove(disk) + os.remove(zip_file) + except OSError as e: + log.error("Can't import config: {}".format(e)) + os.remove(zip_file) + finally: + shutil.rmtree(config_dir, ignore_errors=True) + + def _disk_interface_options(self, disk, disk_index, interface, format=None): + options = [] + extra_drive_options = "" + if format: + extra_drive_options += ",format={}".format(format) + + # From Qemu man page: if the filename contains comma, you must double it + # (for instance, "file=my,,file" to use file "my,file"). + disk = disk.replace(",", ",,") + + if interface == "sata": + # special case, sata controller doesn't exist in Qemu + options.extend(["-device", 'ahci,id=ahci{}'.format(disk_index)]) + options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)]) + options.extend(["-device", 'ide-drive,drive=drive{},bus=ahci{}.0,id=drive{}'.format(disk_index, disk_index, disk_index)]) + elif interface == "nvme": + options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)]) + options.extend(["-device", 'nvme,drive=drive{},serial={}'.format(disk_index, disk_index)]) + elif interface == "scsi": + options.extend(["-device", 'virtio-scsi-pci,id=scsi{}'.format(disk_index)]) + options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk{}'.format(disk, disk_index, disk_index, extra_drive_options)]) + options.extend(["-device", 'scsi-hd,drive=drive{}'.format(disk_index)]) + #elif interface == "sd": + # options.extend(["-drive", 'file={},id=drive{},index={}{}'.format(disk, disk_index, disk_index, extra_drive_options)]) + # options.extend(["-device", 'sd-card,drive=drive{},id=drive{}'.format(disk_index, disk_index, disk_index)]) + else: + options.extend(["-drive", 'file={},if={},index={},media=disk,id=drive{}{}'.format(disk, interface, disk_index, disk_index, extra_drive_options)]) + return options + async def _disk_options(self): options = [] qemu_img_path = self._get_qemu_img() @@ -1691,27 +1799,33 @@ class QemuVM(BaseNode): else: disk = disk_image - # From Qemu man page: if the filename contains comma, you must double it - # (for instance, "file=my,,file" to use file "my,file"). - disk = disk.replace(",", ",,") + options.extend(self._disk_interface_options(disk, disk_index, interface)) - if interface == "sata": - # special case, sata controller doesn't exist in Qemu - options.extend(["-device", 'ahci,id=ahci{}'.format(disk_index)]) - options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)]) - options.extend(["-device", 'ide-drive,drive=drive{},bus=ahci{}.0,id=drive{}'.format(disk_index, disk_index, disk_index)]) - elif interface == "nvme": - options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)]) - options.extend(["-device", 'nvme,drive=drive{},serial={}'.format(disk_index, disk_index)]) - elif interface == "scsi": - options.extend(["-device", 'virtio-scsi-pci,id=scsi{}'.format(disk_index)]) - options.extend(["-drive", 'file={},if=none,id=drive{},index={},media=disk'.format(disk, disk_index, disk_index)]) - options.extend(["-device", 'scsi-hd,drive=drive{}'.format(disk_index)]) - #elif interface == "sd": - # options.extend(["-drive", 'file={},id=drive{},index={}'.format(disk, disk_index, disk_index)]) - # options.extend(["-device", 'sd-card,drive=drive{},id=drive{}'.format(disk_index, disk_index, disk_index)]) + # config disk + disk_image = getattr(self, "config_disk_image") + if disk_image: + if getattr(self, "_hdd_disk_image"): + log.warning("Config disk: blocked by disk image 'hdd'") else: - options.extend(["-drive", 'file={},if={},index={},media=disk,id=drive{}'.format(disk, interface, disk_index, disk_index)]) + disk_name = getattr(self, "config_disk_name") + disk = os.path.join(self.working_dir, disk_name) + interface = getattr(self, "hda_disk_interface", "ide") + await self._import_config() + if not os.path.exists(disk): + try: + shutil.copyfile(disk_image, disk) + except OSError as e: + raise QemuError("Could not create '{}' disk image: {}".format(disk_name, e)) + mtoolsrc = os.path.join(self.working_dir, "mtoolsrc") + if not os.path.exists(mtoolsrc): + try: + with open(mtoolsrc, 'w') as outfile: + outfile.write('drive x:\n') + outfile.write(' file="{}"\n'.format(disk)) + outfile.write(' partition=1\n') + except OSError as e: + raise QemuError("Could not create 'mtoolsrc': {}".format(e)) + options.extend(self._disk_interface_options(disk, 3, interface, "raw")) return options From 99d9728360b69fc34f2cdad00563d04b5d245ac7 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Tue, 7 Apr 2020 14:11:00 +0200 Subject: [PATCH 02/17] QEMU config disk - preserve file timestamp on zip unpack (cherry picked from commit 5c4426847602ab59475403901cf6f3ea3a3e6270) --- gns3server/compute/qemu/qemu_vm.py | 9 ++-- gns3server/compute/qemu/utils/ziputils.py | 53 +++++++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 gns3server/compute/qemu/utils/ziputils.py diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 1eeeb8c0..4c82d658 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -37,6 +37,7 @@ from gns3server.utils import parse_version, shlex_quote from gns3server.utils.asyncio import subprocess_check_output, cancellable_wait_run_in_executor from .qemu_error import QemuError from .utils.qcow2 import Qcow2, Qcow2Error +from .utils.ziputils import pack_zip, unpack_zip from ..adapters.ethernet_adapter import EthernetAdapter from ..error import NodeError, ImageMissingError from ..nios.nio_udp import NIOUDP @@ -1675,8 +1676,8 @@ class QemuVM(BaseNode): os.mkdir(config_dir) if os.path.exists(zip_file): os.remove(zip_file) - if await self._mcopy("-s", "-m", "x:/", config_dir) == 0: - shutil.make_archive(zip_file[:-4], "zip", config_dir) + if await self._mcopy("-s", "-m", "-n", "--", "x:/", config_dir) == 0: + pack_zip(zip_file, config_dir) except OSError as e: log.error("Can't export config: {}".format(e)) finally: @@ -1692,12 +1693,12 @@ class QemuVM(BaseNode): try: shutil.rmtree(config_dir, ignore_errors=True) os.mkdir(config_dir) - shutil.unpack_archive(zip_file, config_dir) + unpack_zip(zip_file, config_dir) shutil.copyfile(getattr(self, "config_disk_image"), disk) config_files = [os.path.join(config_dir, fname) for fname in os.listdir(config_dir)] if config_files: - if await self._mcopy("-s", "-m", *config_files, "x:/") != 0: + if await self._mcopy("-s", "-m", "-o", "--", *config_files, "x:/") != 0: os.remove(disk) os.remove(zip_file) except OSError as e: diff --git a/gns3server/compute/qemu/utils/ziputils.py b/gns3server/compute/qemu/utils/ziputils.py new file mode 100644 index 00000000..3ff8c999 --- /dev/null +++ b/gns3server/compute/qemu/utils/ziputils.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import time +import shutil +import zipfile + +def pack_zip(filename, root_dir=None, base_dir=None): + """Create a zip archive""" + + if filename[-4:].lower() == ".zip": + filename = filename[:-4] + shutil.make_archive(filename, 'zip', root_dir, base_dir) + +def unpack_zip(filename, extract_dir=None): + """Unpack a zip archive""" + + dirs = [] + if not extract_dir: + extract_dir = os.getcwd() + + try: + with zipfile.ZipFile(filename, 'r') as zfile: + for zinfo in zfile.infolist(): + fname = os.path.join(extract_dir, zinfo.filename) + date_time = time.mktime(zinfo.date_time + (0, 0, -1)) + zfile.extract(zinfo, extract_dir) + + # update timestamp + if zinfo.is_dir(): + dirs.append((fname, date_time)) + else: + os.utime(fname, (date_time, date_time)) + # update timestamp of directories + for fname, date_time in reversed(dirs): + os.utime(fname, (date_time, date_time)) + except zipfile.BadZipFile: + raise shutil.ReadError("%s is not a zip file" % filename) From 0db0f6256bf396973d3b511f8b31841edcb0e43f Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Wed, 15 Apr 2020 20:50:59 +0200 Subject: [PATCH 03/17] QEMU config disk - get rid of mtoolsrc (cherry picked from commit 450c6cddc743c5b1a1bfa4a9b58a8aaa4983160c) --- gns3server/compute/qemu/qemu_vm.py | 43 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 4c82d658..78d54fc8 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -26,6 +26,7 @@ import re import shlex import math import shutil +import struct import asyncio import socket import gns3server @@ -1646,11 +1647,26 @@ class QemuVM(BaseNode): log.info("{} returned with {}".format(self._get_qemu_img(), retcode)) return retcode - async def _mcopy(self, *args): - env = os.environ - env["MTOOLSRC"] = 'mtoolsrc' + async def _mcopy(self, image, *args): try: - process = await asyncio.create_subprocess_exec("mcopy", *args, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.working_dir, env=env) + # read offset of first partition from MBR + with open(image, "rb") as img_file: + mbr = img_file.read(512) + part_type, offset, signature = struct.unpack("<450xB3xL52xH", mbr) + if signature != 0xAA55: + log.error("mcopy failure: {}: invalid MBR".format(image)) + return 1 + if part_type not in (1, 4, 6, 11, 12, 14): + log.error("mcopy failure: {}: invalid partition type {:02X}" + .format(image, part_type)) + return 1 + part_image = image + "@@{}S".format(offset) + + process = await asyncio.create_subprocess_exec( + "mcopy", "-i", part_image, *args, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + cwd=self.working_dir) (stdout, _) = await process.communicate() retcode = process.returncode except (OSError, subprocess.SubprocessError) as e: @@ -1666,8 +1682,10 @@ class QemuVM(BaseNode): async def _export_config(self): disk_name = getattr(self, "config_disk_name") - if not disk_name or \ - not os.path.exists(os.path.join(self.working_dir, disk_name)): + if not disk_name: + return + disk = os.path.join(self.working_dir, disk_name) + if not os.path.exists(disk): return config_dir = os.path.join(self.working_dir, "configs") zip_file = os.path.join(self.working_dir, "config.zip") @@ -1676,7 +1694,7 @@ class QemuVM(BaseNode): os.mkdir(config_dir) if os.path.exists(zip_file): os.remove(zip_file) - if await self._mcopy("-s", "-m", "-n", "--", "x:/", config_dir) == 0: + if await self._mcopy(disk, "-s", "-m", "-n", "--", "::/", config_dir) == 0: pack_zip(zip_file, config_dir) except OSError as e: log.error("Can't export config: {}".format(e)) @@ -1698,7 +1716,7 @@ class QemuVM(BaseNode): config_files = [os.path.join(config_dir, fname) for fname in os.listdir(config_dir)] if config_files: - if await self._mcopy("-s", "-m", "-o", "--", *config_files, "x:/") != 0: + if await self._mcopy(disk, "-s", "-m", "-o", "--", *config_files, "::/") != 0: os.remove(disk) os.remove(zip_file) except OSError as e: @@ -1817,15 +1835,6 @@ class QemuVM(BaseNode): shutil.copyfile(disk_image, disk) except OSError as e: raise QemuError("Could not create '{}' disk image: {}".format(disk_name, e)) - mtoolsrc = os.path.join(self.working_dir, "mtoolsrc") - if not os.path.exists(mtoolsrc): - try: - with open(mtoolsrc, 'w') as outfile: - outfile.write('drive x:\n') - outfile.write(' file="{}"\n'.format(disk)) - outfile.write(' partition=1\n') - except OSError as e: - raise QemuError("Could not create 'mtoolsrc': {}".format(e)) options.extend(self._disk_interface_options(disk, 3, interface, "raw")) return options From 347035a99bf99618d945105d4205dc3d985eceb5 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Thu, 16 Apr 2020 11:07:56 +0200 Subject: [PATCH 04/17] QEMU config disk - add missing config disk to image directory (cherry picked from commit 2e0fba925bdd796ddd5eea0a4c9e4dcebed861ab) --- gns3server/compute/qemu/qemu_vm.py | 27 +++++++++++++----- .../compute/qemu/resources/config.img.zip | Bin 0 -> 1368 bytes 2 files changed, 20 insertions(+), 7 deletions(-) create mode 100644 gns3server/compute/qemu/resources/config.img.zip diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 78d54fc8..32608370 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -46,6 +46,7 @@ from ..nios.nio_tap import NIOTAP from ..base_node import BaseNode from ...schemas.qemu import QEMU_OBJECT_SCHEMA, QEMU_PLATFORMS from ...utils.asyncio import monitor_process +from ...utils.get_resource import get_resource from ...utils.images import md5sum from ...utils import macaddress_to_int, int_to_macaddress @@ -130,19 +131,31 @@ class QemuVM(BaseNode): self.adapters = 1 # creates 1 adapter by default # config disk - self.config_disk_name = "config.img" + config_disk_name = "config.img" + self.config_disk_name = "" + self.config_disk_image = "" if not shutil.which("mcopy"): log.warning("Config disk: 'mtools' are not installed.") - self.config_disk_name = "" - self.config_disk_image = "" else: try: self.config_disk_image = self.manager.get_abs_image_path( - self.config_disk_name, self.project.path) + config_disk_name, self.project.path) + self.config_disk_name = config_disk_name except (NodeError, ImageMissingError) as e: - log.warning("Config disk: {}".format(e)) - self.config_disk_name = "" - self.config_disk_image = "" + config_disk_zip = get_resource("compute/qemu/resources/{}.zip" + .format(config_disk_name)) + if config_disk_zip and os.path.exists(config_disk_zip): + directory = self.manager.get_images_directory() + try: + unpack_zip(config_disk_zip, directory) + self.config_disk_image = os.path.join(directory, + config_disk_name) + self.config_disk_name = config_disk_name + except OSError as e: + log.warning("Config disk creation: {}".format(e)) + else: + log.warning("Config disk: image '{}' missing" + .format(config_disk_name)) log.info('QEMU VM "{name}" [{id}] has been created'.format(name=self._name, id=self._id)) diff --git a/gns3server/compute/qemu/resources/config.img.zip b/gns3server/compute/qemu/resources/config.img.zip new file mode 100644 index 0000000000000000000000000000000000000000..7ba43f9e0db1535a2d465dad484dbb05046d0575 GIT binary patch literal 1368 zcmWIWW@Zs#U|`^2;D~Dru#K9RvW^AFWe{NCVvu1-&d*EBOxMfIO%Dy>WMF>qt1IsH zrM|e*3T_5QmKV$n3}E8zbwxksKmmpeKW95!pUTp@`shWc00pfBTeG9qHMp<8A#4)E zbXhoHmV>Q` L1Az1=aQO)Uu Date: Wed, 17 Jun 2020 17:06:55 +0200 Subject: [PATCH 05/17] QEMU config disk - use disk interface of HD-D, fallback is HD-A (cherry picked from commit b672900406e5ce10251cc1e231fee1d117ca3805) --- gns3server/compute/qemu/qemu_vm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 32608370..6c6f10f3 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1841,7 +1841,9 @@ class QemuVM(BaseNode): else: disk_name = getattr(self, "config_disk_name") disk = os.path.join(self.working_dir, disk_name) - interface = getattr(self, "hda_disk_interface", "ide") + interface = getattr(self, "hdd_disk_interface", "ide") + if interface == "ide": + interface = getattr(self, "hda_disk_interface", "none") await self._import_config() if not os.path.exists(disk): try: From f747b3a880180e480db32e8ee595c652bbe2896f Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Sun, 28 Jun 2020 09:21:57 +0200 Subject: [PATCH 06/17] QEMU config disk - notification of import/export errors (cherry picked from commit 50c49cfedb226c3d15397fec443d86ebc0fdb26a) --- gns3server/compute/qemu/qemu_vm.py | 32 ++++++++++++++---------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 6c6f10f3..923c691c 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1667,12 +1667,10 @@ class QemuVM(BaseNode): mbr = img_file.read(512) part_type, offset, signature = struct.unpack("<450xB3xL52xH", mbr) if signature != 0xAA55: - log.error("mcopy failure: {}: invalid MBR".format(image)) - return 1 + raise OSError("mcopy failure: {}: invalid MBR".format(image)) if part_type not in (1, 4, 6, 11, 12, 14): - log.error("mcopy failure: {}: invalid partition type {:02X}" - .format(image, part_type)) - return 1 + raise OSError("mcopy failure: {}: invalid partition type {:02X}" + .format(image, part_type)) part_image = image + "@@{}S".format(offset) process = await asyncio.create_subprocess_exec( @@ -1683,15 +1681,13 @@ class QemuVM(BaseNode): (stdout, _) = await process.communicate() retcode = process.returncode except (OSError, subprocess.SubprocessError) as e: - log.error("mcopy failure: {}".format(e)) - return 1 + raise OSError("mcopy failure: {}".format(e)) if retcode != 0: stdout = stdout.decode("utf-8").rstrip() if stdout: - log.error("mcopy failure: {}".format(stdout)) + raise OSError("mcopy failure: {}".format(stdout)) else: - log.error("mcopy failure: return code {}".format(retcode)) - return retcode + raise OSError("mcopy failure: return code {}".format(retcode)) async def _export_config(self): disk_name = getattr(self, "config_disk_name") @@ -1707,10 +1703,11 @@ class QemuVM(BaseNode): os.mkdir(config_dir) if os.path.exists(zip_file): os.remove(zip_file) - if await self._mcopy(disk, "-s", "-m", "-n", "--", "::/", config_dir) == 0: - pack_zip(zip_file, config_dir) + await self._mcopy(disk, "-s", "-m", "-n", "--", "::/", config_dir) + pack_zip(zip_file, config_dir) except OSError as e: - log.error("Can't export config: {}".format(e)) + log.warning("Can't export config: {}".format(e)) + self.project.emit("log.warning", {"message": "{}: Can't export config: {}".format(self._name, e)}) finally: shutil.rmtree(config_dir, ignore_errors=True) @@ -1729,11 +1726,12 @@ class QemuVM(BaseNode): config_files = [os.path.join(config_dir, fname) for fname in os.listdir(config_dir)] if config_files: - if await self._mcopy(disk, "-s", "-m", "-o", "--", *config_files, "::/") != 0: - os.remove(disk) - os.remove(zip_file) + await self._mcopy(disk, "-s", "-m", "-o", "--", *config_files, "::/") except OSError as e: - log.error("Can't import config: {}".format(e)) + log.warning("Can't import config: {}".format(e)) + self.project.emit("log.warning", {"message": "{}: Can't import config: {}".format(self._name, e)}) + if os.path.exists(disk): + os.remove(disk) os.remove(zip_file) finally: shutil.rmtree(config_dir, ignore_errors=True) From 053828f3e8f3d350a7143a0c6543feb2a0024830 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Sun, 28 Jun 2020 16:35:39 +0200 Subject: [PATCH 07/17] QEMU config disk - init config disk in base class (cherry picked from commit 2bbee15b18594ee517046016c6c33e066b300659) --- gns3server/compute/qemu/__init__.py | 25 +++++++++++++++++++++++++ gns3server/compute/qemu/qemu_vm.py | 25 ++++++------------------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/gns3server/compute/qemu/__init__.py b/gns3server/compute/qemu/__init__.py index 828e94b2..663ee37f 100644 --- a/gns3server/compute/qemu/__init__.py +++ b/gns3server/compute/qemu/__init__.py @@ -27,10 +27,13 @@ import re import subprocess from ...utils.asyncio import subprocess_check_output +from ...utils.get_resource import get_resource from ..base_manager import BaseManager +from ..error import NodeError, ImageMissingError from .qemu_error import QemuError from .qemu_vm import QemuVM from .utils.guest_cid import get_next_guest_cid +from .utils.ziputils import unpack_zip import logging log = logging.getLogger(__name__) @@ -45,6 +48,7 @@ class Qemu(BaseManager): super().__init__() self._guest_cid_lock = asyncio.Lock() + self._init_config_disk() async def create_node(self, *args, **kwargs): """ @@ -343,3 +347,24 @@ class Qemu(BaseManager): log.info("Qemu disk '{}' extended by {} MB".format(path, extend)) except (OSError, subprocess.SubprocessError) as e: raise QemuError("Could not update disk image {}:{}".format(path, e)) + + def _init_config_disk(self): + """ + Initialize the default config disk + """ + + self.config_disk = "config.img" + try: + self.get_abs_image_path(self.config_disk) + except (NodeError, ImageMissingError) as e: + config_disk_zip = get_resource("compute/qemu/resources/{}.zip" + .format(self.config_disk)) + if config_disk_zip and os.path.exists(config_disk_zip): + directory = self.get_images_directory() + try: + unpack_zip(config_disk_zip, directory) + except OSError as e: + log.warning("Config disk creation: {}".format(e)) + else: + log.warning("Config disk: image '{}' missing" + .format(self.config_disk)) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 923c691c..8380dcdc 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -46,7 +46,6 @@ from ..nios.nio_tap import NIOTAP from ..base_node import BaseNode from ...schemas.qemu import QEMU_OBJECT_SCHEMA, QEMU_PLATFORMS from ...utils.asyncio import monitor_process -from ...utils.get_resource import get_resource from ...utils.images import md5sum from ...utils import macaddress_to_int, int_to_macaddress @@ -131,31 +130,19 @@ class QemuVM(BaseNode): self.adapters = 1 # creates 1 adapter by default # config disk - config_disk_name = "config.img" - self.config_disk_name = "" + self.config_disk_name = self.manager.config_disk self.config_disk_image = "" if not shutil.which("mcopy"): log.warning("Config disk: 'mtools' are not installed.") + self.config_disk_name = "" else: try: self.config_disk_image = self.manager.get_abs_image_path( - config_disk_name, self.project.path) - self.config_disk_name = config_disk_name + self.config_disk_name) except (NodeError, ImageMissingError) as e: - config_disk_zip = get_resource("compute/qemu/resources/{}.zip" - .format(config_disk_name)) - if config_disk_zip and os.path.exists(config_disk_zip): - directory = self.manager.get_images_directory() - try: - unpack_zip(config_disk_zip, directory) - self.config_disk_image = os.path.join(directory, - config_disk_name) - self.config_disk_name = config_disk_name - except OSError as e: - log.warning("Config disk creation: {}".format(e)) - else: - log.warning("Config disk: image '{}' missing" - .format(config_disk_name)) + log.warning("Config disk: image '{}' missing" + .format(self.config_disk_name)) + self.config_disk_name = "" log.info('QEMU VM "{name}" [{id}] has been created'.format(name=self._name, id=self._id)) From 9acb2ceda1a2f9ad6f8e7bd6229410a47225f0bb Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Fri, 3 Jul 2020 11:31:17 +0200 Subject: [PATCH 08/17] QEMU config disk - improve error handling (cherry picked from commit 068c31038f33e08d5bcf1b71a3b7eae0133e29dd) --- gns3server/compute/qemu/qemu_vm.py | 31 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 8380dcdc..d33f7316 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1686,17 +1686,15 @@ class QemuVM(BaseNode): config_dir = os.path.join(self.working_dir, "configs") zip_file = os.path.join(self.working_dir, "config.zip") try: - shutil.rmtree(config_dir, ignore_errors=True) os.mkdir(config_dir) + await self._mcopy(disk, "-s", "-m", "-n", "--", "::/", config_dir) if os.path.exists(zip_file): os.remove(zip_file) - await self._mcopy(disk, "-s", "-m", "-n", "--", "::/", config_dir) pack_zip(zip_file, config_dir) except OSError as e: log.warning("Can't export config: {}".format(e)) self.project.emit("log.warning", {"message": "{}: Can't export config: {}".format(self._name, e)}) - finally: - shutil.rmtree(config_dir, ignore_errors=True) + shutil.rmtree(config_dir, ignore_errors=True) async def _import_config(self): disk_name = getattr(self, "config_disk_name") @@ -1705,23 +1703,23 @@ class QemuVM(BaseNode): return config_dir = os.path.join(self.working_dir, "configs") disk = os.path.join(self.working_dir, disk_name) + disk_tmp = disk + ".tmp" try: - shutil.rmtree(config_dir, ignore_errors=True) os.mkdir(config_dir) + shutil.copyfile(getattr(self, "config_disk_image"), disk_tmp) unpack_zip(zip_file, config_dir) - shutil.copyfile(getattr(self, "config_disk_image"), disk) config_files = [os.path.join(config_dir, fname) for fname in os.listdir(config_dir)] if config_files: - await self._mcopy(disk, "-s", "-m", "-o", "--", *config_files, "::/") + await self._mcopy(disk_tmp, "-s", "-m", "-o", "--", *config_files, "::/") + os.replace(disk_tmp, disk) except OSError as e: log.warning("Can't import config: {}".format(e)) self.project.emit("log.warning", {"message": "{}: Can't import config: {}".format(self._name, e)}) - if os.path.exists(disk): - os.remove(disk) - os.remove(zip_file) - finally: - shutil.rmtree(config_dir, ignore_errors=True) + if os.path.exists(disk_tmp): + os.remove(disk_tmp) + os.remove(zip_file) + shutil.rmtree(config_dir, ignore_errors=True) def _disk_interface_options(self, disk, disk_index, interface, format=None): options = [] @@ -1830,12 +1828,15 @@ class QemuVM(BaseNode): if interface == "ide": interface = getattr(self, "hda_disk_interface", "none") await self._import_config() - if not os.path.exists(disk): + disk_exists = os.path.exists(disk) + if not disk_exists: try: shutil.copyfile(disk_image, disk) + disk_exists = True except OSError as e: - raise QemuError("Could not create '{}' disk image: {}".format(disk_name, e)) - options.extend(self._disk_interface_options(disk, 3, interface, "raw")) + log.warning("Could not create '{}' disk image: {}".format(disk_name, e)) + if disk_exists: + options.extend(self._disk_interface_options(disk, 3, interface, "raw")) return options From c684c554bf90f831f5fc6305e44dd14374413831 Mon Sep 17 00:00:00 2001 From: grossmj Date: Thu, 13 Aug 2020 17:10:31 +0930 Subject: [PATCH 09/17] Fix tests (cherry picked from commit 2ba6eac1135e0ddf01cb2d975822303d258c5299) --- gns3server/compute/qemu/__init__.py | 10 ++++------ gns3server/compute/qemu/qemu_vm.py | 20 ++++++++++---------- tests/compute/qemu/test_qemu_vm.py | 1 + tests/compute/test_manager.py | 3 ++- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/gns3server/compute/qemu/__init__.py b/gns3server/compute/qemu/__init__.py index 663ee37f..bca490cc 100644 --- a/gns3server/compute/qemu/__init__.py +++ b/gns3server/compute/qemu/__init__.py @@ -48,6 +48,7 @@ class Qemu(BaseManager): super().__init__() self._guest_cid_lock = asyncio.Lock() + self.config_disk = "config.img" self._init_config_disk() async def create_node(self, *args, **kwargs): @@ -353,12 +354,10 @@ class Qemu(BaseManager): Initialize the default config disk """ - self.config_disk = "config.img" try: self.get_abs_image_path(self.config_disk) - except (NodeError, ImageMissingError) as e: - config_disk_zip = get_resource("compute/qemu/resources/{}.zip" - .format(self.config_disk)) + except (NodeError, ImageMissingError): + config_disk_zip = get_resource("compute/qemu/resources/{}.zip".format(self.config_disk)) if config_disk_zip and os.path.exists(config_disk_zip): directory = self.get_images_directory() try: @@ -366,5 +365,4 @@ class Qemu(BaseManager): except OSError as e: log.warning("Config disk creation: {}".format(e)) else: - log.warning("Config disk: image '{}' missing" - .format(self.config_disk)) + log.warning("Config disk: image '{}' missing".format(self.config_disk)) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index d33f7316..2870ad29 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -132,17 +132,17 @@ class QemuVM(BaseNode): # config disk self.config_disk_name = self.manager.config_disk self.config_disk_image = "" - if not shutil.which("mcopy"): - log.warning("Config disk: 'mtools' are not installed.") - self.config_disk_name = "" - else: - try: - self.config_disk_image = self.manager.get_abs_image_path( - self.config_disk_name) - except (NodeError, ImageMissingError) as e: - log.warning("Config disk: image '{}' missing" - .format(self.config_disk_name)) + if self.config_disk_name: + if not shutil.which("mcopy"): + log.warning("Config disk: 'mtools' are not installed.") self.config_disk_name = "" + else: + try: + self.config_disk_image = self.manager.get_abs_image_path(self.config_disk_name) + except (NodeError, ImageMissingError) as e: + log.warning("Config disk: image '{}' missing" + .format(self.config_disk_name)) + self.config_disk_name = "" log.info('QEMU VM "{name}" [{id}] has been created'.format(name=self._name, id=self._id)) diff --git a/tests/compute/qemu/test_qemu_vm.py b/tests/compute/qemu/test_qemu_vm.py index 6067a205..2df4c389 100644 --- a/tests/compute/qemu/test_qemu_vm.py +++ b/tests/compute/qemu/test_qemu_vm.py @@ -337,6 +337,7 @@ def test_set_qemu_path_kvm_binary(vm, fake_qemu_binary): async def test_set_platform(compute_project, manager): + manager.config_disk = None # avoids conflict with config.img support with patch("shutil.which", return_value="/bin/qemu-system-x86_64") as which_mock: with patch("gns3server.compute.qemu.QemuVM._check_qemu_path"): vm = QemuVM("test", "00010203-0405-0607-0809-0a0b0c0d0e0f", compute_project, manager, platform="x86_64") diff --git a/tests/compute/test_manager.py b/tests/compute/test_manager.py index bceb9e20..e257e763 100644 --- a/tests/compute/test_manager.py +++ b/tests/compute/test_manager.py @@ -18,7 +18,7 @@ import uuid import os import pytest -from unittest.mock import patch +from unittest.mock import patch, MagicMock from tests.utils import asyncio_patch from gns3server.compute.vpcs import VPCS @@ -41,6 +41,7 @@ async def vpcs(loop, port_manager): async def qemu(loop, port_manager): Qemu._instance = None + Qemu._init_config_disk = MagicMock() # do not create the config.img image qemu = Qemu.instance() qemu.port_manager = port_manager return qemu From 9d3f7c79a2564611ec4144e683ebbde9e16ac59e Mon Sep 17 00:00:00 2001 From: grossmj Date: Thu, 13 Aug 2020 17:18:45 +0930 Subject: [PATCH 10/17] Fix more tests (cherry picked from commit 546982d1ea1d44ce97bc974b65248389c29cf80e) --- tests/conftest.py | 4 ++-- tests/handlers/api/compute/test_qemu.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index d94b4bc2..7311b7e6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -117,8 +117,8 @@ def images_dir(config): path = config.get_section_config("Server").get("images_path") os.makedirs(path, exist_ok=True) - os.makedirs(os.path.join(path, "QEMU")) - os.makedirs(os.path.join(path, "IOU")) + os.makedirs(os.path.join(path, "QEMU"), exist_ok=True) + os.makedirs(os.path.join(path, "IOU"), exist_ok=True) return path diff --git a/tests/handlers/api/compute/test_qemu.py b/tests/handlers/api/compute/test_qemu.py index d88ee33d..112cdbda 100644 --- a/tests/handlers/api/compute/test_qemu.py +++ b/tests/handlers/api/compute/test_qemu.py @@ -277,7 +277,8 @@ async def test_images(compute_api, fake_qemu_vm): response = await compute_api.get("/qemu/images") assert response.status == 200 - assert response.json == [{"filename": "linux载.img", "path": "linux载.img", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}] + assert response.json == [{'filename': 'config.img', 'filesize': 1048576, 'md5sum': '0ab49056760ae1db6c25376446190b47', 'path': 'config.img'}, + {"filename": "linux载.img", "path": "linux载.img", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}] @pytest.mark.skipif(sys.platform.startswith("win"), reason="Does not work on Windows") From a56b816c1af2b0edaa5d6f64f86ea7665be8abde Mon Sep 17 00:00:00 2001 From: grossmj Date: Fri, 14 Aug 2020 17:57:24 +0930 Subject: [PATCH 11/17] Add explicit option to automatically create or not the config disk. Off by default. (cherry picked from commit 56aba96a5fb57f502b283a0bc543da415bb3943a) --- gns3server/compute/qemu/qemu_vm.py | 33 +++++++++++++++++++++++++---- gns3server/schemas/qemu.py | 13 ++++++++++++ gns3server/schemas/qemu_template.py | 5 +++++ 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 2870ad29..3e7b0280 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -122,6 +122,7 @@ class QemuVM(BaseNode): self._kernel_command_line = "" self._legacy_networking = False self._replicate_network_connection_state = True + self._create_config_disk = False self._on_close = "power_off" self._cpu_throttling = 0 # means no CPU throttling self._process_priority = "low" @@ -139,9 +140,8 @@ class QemuVM(BaseNode): else: try: self.config_disk_image = self.manager.get_abs_image_path(self.config_disk_name) - except (NodeError, ImageMissingError) as e: - log.warning("Config disk: image '{}' missing" - .format(self.config_disk_name)) + except (NodeError, ImageMissingError): + log.warning("Config disk: image '{}' missing".format(self.config_disk_name)) self.config_disk_name = "" log.info('QEMU VM "{name}" [{id}] has been created'.format(name=self._name, id=self._id)) @@ -671,6 +671,30 @@ class QemuVM(BaseNode): log.info('QEMU VM "{name}" [{id}] has disabled network connection state replication'.format(name=self._name, id=self._id)) self._replicate_network_connection_state = replicate_network_connection_state + @property + def create_config_disk(self): + """ + Returns whether a config disk is automatically created on HDD disk interface (secondary slave) + + :returns: boolean + """ + + return self._create_config_disk + + @create_config_disk.setter + def create_config_disk(self, create_config_disk): + """ + Sets whether a config disk is automatically created on HDD disk interface (secondary slave) + + :param replicate_network_connection_state: boolean + """ + + if create_config_disk: + log.info('QEMU VM "{name}" [{id}] has enabled the config disk creation feature'.format(name=self._name, id=self._id)) + else: + log.info('QEMU VM "{name}" [{id}] has disabled the config disk creation feature'.format(name=self._name, id=self._id)) + self._create_config_disk = create_config_disk + @property def on_close(self): """ @@ -1818,7 +1842,7 @@ class QemuVM(BaseNode): # config disk disk_image = getattr(self, "config_disk_image") - if disk_image: + if disk_image and self._create_config_disk: if getattr(self, "_hdd_disk_image"): log.warning("Config disk: blocked by disk image 'hdd'") else: @@ -1837,6 +1861,7 @@ class QemuVM(BaseNode): log.warning("Could not create '{}' disk image: {}".format(disk_name, e)) if disk_exists: options.extend(self._disk_interface_options(disk, 3, interface, "raw")) + self.hdd_disk_image = disk return options diff --git a/gns3server/schemas/qemu.py b/gns3server/schemas/qemu.py index 567e5c3f..3ae444d4 100644 --- a/gns3server/schemas/qemu.py +++ b/gns3server/schemas/qemu.py @@ -190,6 +190,10 @@ QEMU_CREATE_SCHEMA = { "description": "Replicate the network connection state for links in Qemu", "type": ["boolean", "null"], }, + "create_config_disk": { + "description": "Automatically create a config disk on HDD disk interface (secondary slave)", + "type": ["boolean", "null"], + }, "on_close": { "description": "Action to execute on the VM is closed", "enum": ["power_off", "shutdown_signal", "save_vm_state"], @@ -380,6 +384,10 @@ QEMU_UPDATE_SCHEMA = { "description": "Replicate the network connection state for links in Qemu", "type": ["boolean", "null"], }, + "create_config_disk": { + "description": "Automatically create a config disk on HDD disk interface (secondary slave)", + "type": ["boolean", "null"], + }, "on_close": { "description": "Action to execute on the VM is closed", "enum": ["power_off", "shutdown_signal", "save_vm_state"], @@ -583,6 +591,10 @@ QEMU_OBJECT_SCHEMA = { "description": "Replicate the network connection state for links in Qemu", "type": "boolean", }, + "create_config_disk": { + "description": "Automatically create a config disk on HDD disk interface (secondary slave)", + "type": ["boolean", "null"], + }, "on_close": { "description": "Action to execute on the VM is closed", "enum": ["power_off", "shutdown_signal", "save_vm_state"], @@ -653,6 +665,7 @@ QEMU_OBJECT_SCHEMA = { "kernel_command_line", "legacy_networking", "replicate_network_connection_state", + "create_config_disk", "on_close", "cpu_throttling", "process_priority", diff --git a/gns3server/schemas/qemu_template.py b/gns3server/schemas/qemu_template.py index f98c81d7..6414066e 100644 --- a/gns3server/schemas/qemu_template.py +++ b/gns3server/schemas/qemu_template.py @@ -183,6 +183,11 @@ QEMU_TEMPLATE_PROPERTIES = { "type": "boolean", "default": True }, + "create_config_disk": { + "description": "Automatically create a config disk on HDD disk interface (secondary slave)", + "type": "boolean", + "default": True + }, "on_close": { "description": "Action to execute on the VM is closed", "enum": ["power_off", "shutdown_signal", "save_vm_state"], From ec02150fd2207d52374ecfe19dd3dd018ef27acb Mon Sep 17 00:00:00 2001 From: grossmj Date: Sat, 15 Aug 2020 16:14:16 +0930 Subject: [PATCH 12/17] Set default disk interface type to "none". Fail-safe: use "ide" if an image is set but no interface type is configured. Use the HDA disk interface type if none has been configured for HDD. (cherry picked from commit 464fd804cebaf3f569d4552ba53b82ab3b87c17c) --- gns3server/compute/qemu/qemu_vm.py | 25 +++++++++++++------------ gns3server/schemas/qemu_template.py | 8 ++++---- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 3e7b0280..8b40396a 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -104,10 +104,10 @@ class QemuVM(BaseNode): self._hdb_disk_image = "" self._hdc_disk_image = "" self._hdd_disk_image = "" - self._hda_disk_interface = "ide" - self._hdb_disk_interface = "ide" - self._hdc_disk_interface = "ide" - self._hdd_disk_interface = "ide" + self._hda_disk_interface = "none" + self._hdb_disk_interface = "none" + self._hdc_disk_interface = "none" + self._hdd_disk_interface = "none" self._cdrom_image = "" self._bios_image = "" self._boot_priority = "c" @@ -1782,13 +1782,15 @@ class QemuVM(BaseNode): for disk_index, drive in enumerate(drives): disk_image = getattr(self, "_hd{}_disk_image".format(drive)) - interface = getattr(self, "hd{}_disk_interface".format(drive)) - if not disk_image: continue - disk_name = "hd" + drive + interface = getattr(self, "hd{}_disk_interface".format(drive)) + # fail-safe: use "ide" if there is a disk image and no interface type has been explicitly configured + if interface == "none": + setattr(self, "hd{}_disk_interface".format(drive), "ide") + disk_name = "hd" + drive if not os.path.isfile(disk_image) or not os.path.exists(disk_image): if os.path.islink(disk_image): raise QemuError("{} disk image '{}' linked to '{}' is not accessible".format(disk_name, disk_image, os.path.realpath(disk_image))) @@ -1848,9 +1850,9 @@ class QemuVM(BaseNode): else: disk_name = getattr(self, "config_disk_name") disk = os.path.join(self.working_dir, disk_name) - interface = getattr(self, "hdd_disk_interface", "ide") - if interface == "ide": - interface = getattr(self, "hda_disk_interface", "none") + if self.hdd_disk_interface == "none": + # use the HDA interface type if none has been configured for HDD + self.hdd_disk_interface = getattr(self, "hda_disk_interface", "none") await self._import_config() disk_exists = os.path.exists(disk) if not disk_exists: @@ -1860,8 +1862,7 @@ class QemuVM(BaseNode): except OSError as e: log.warning("Could not create '{}' disk image: {}".format(disk_name, e)) if disk_exists: - options.extend(self._disk_interface_options(disk, 3, interface, "raw")) - self.hdd_disk_image = disk + options.extend(self._disk_interface_options(disk, 3, self.hdd_disk_interface, "raw")) return options diff --git a/gns3server/schemas/qemu_template.py b/gns3server/schemas/qemu_template.py index 6414066e..4202f66c 100644 --- a/gns3server/schemas/qemu_template.py +++ b/gns3server/schemas/qemu_template.py @@ -116,7 +116,7 @@ QEMU_TEMPLATE_PROPERTIES = { "hda_disk_interface": { "description": "QEMU hda interface", "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"], - "default": "ide" + "default": "none" }, "hdb_disk_image": { "description": "QEMU hdb disk image path", @@ -126,7 +126,7 @@ QEMU_TEMPLATE_PROPERTIES = { "hdb_disk_interface": { "description": "QEMU hdb interface", "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"], - "default": "ide" + "default": "none" }, "hdc_disk_image": { "description": "QEMU hdc disk image path", @@ -136,7 +136,7 @@ QEMU_TEMPLATE_PROPERTIES = { "hdc_disk_interface": { "description": "QEMU hdc interface", "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"], - "default": "ide" + "default": "none" }, "hdd_disk_image": { "description": "QEMU hdd disk image path", @@ -146,7 +146,7 @@ QEMU_TEMPLATE_PROPERTIES = { "hdd_disk_interface": { "description": "QEMU hdd interface", "enum": ["ide", "sata", "nvme", "scsi", "sd", "mtd", "floppy", "pflash", "virtio", "none"], - "default": "ide" + "default": "none" }, "cdrom_image": { "description": "QEMU cdrom image path", From f2ddef855faedaed9d4077c9cd6c924d907ed0a3 Mon Sep 17 00:00:00 2001 From: grossmj Date: Sat, 15 Aug 2020 16:35:31 +0930 Subject: [PATCH 13/17] Fix tests. (cherry picked from commit 620d93634e835701c271dd70cbe2abf9aa16b1f4) --- gns3server/compute/qemu/qemu_vm.py | 3 ++- tests/handlers/api/controller/test_template.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 8b40396a..62148042 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1788,7 +1788,8 @@ class QemuVM(BaseNode): interface = getattr(self, "hd{}_disk_interface".format(drive)) # fail-safe: use "ide" if there is a disk image and no interface type has been explicitly configured if interface == "none": - setattr(self, "hd{}_disk_interface".format(drive), "ide") + interface = "ide" + setattr(self, "hd{}_disk_interface".format(drive), interface) disk_name = "hd" + drive if not os.path.isfile(disk_image) or not os.path.exists(disk_image): diff --git a/tests/handlers/api/controller/test_template.py b/tests/handlers/api/controller/test_template.py index c9eaf75a..2aeff91f 100644 --- a/tests/handlers/api/controller/test_template.py +++ b/tests/handlers/api/controller/test_template.py @@ -669,13 +669,13 @@ async def test_qemu_template_create(controller_api): "default_name_format": "{name}-{0}", "first_port_name": "", "hda_disk_image": "IOSvL2-15.2.4.0.55E.qcow2", - "hda_disk_interface": "ide", + "hda_disk_interface": "none", "hdb_disk_image": "", - "hdb_disk_interface": "ide", + "hdb_disk_interface": "none", "hdc_disk_image": "", - "hdc_disk_interface": "ide", + "hdc_disk_interface": "none", "hdd_disk_image": "", - "hdd_disk_interface": "ide", + "hdd_disk_interface": "none", "initrd": "", "kernel_command_line": "", "kernel_image": "", From 01db2d2a866a4e8ba413ff661b61db284dc6896b Mon Sep 17 00:00:00 2001 From: Jeremy Grossmann Date: Mon, 17 Aug 2020 12:45:57 +0930 Subject: [PATCH 14/17] Create config disk property false by default for Qemu templates Ref https://github.com/GNS3/gns3-gui/issues/3035 (cherry picked from commit a2e884e315ca25432ba5612142d60ce8e1dd7911) --- gns3server/schemas/qemu_template.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gns3server/schemas/qemu_template.py b/gns3server/schemas/qemu_template.py index 4202f66c..496ee06a 100644 --- a/gns3server/schemas/qemu_template.py +++ b/gns3server/schemas/qemu_template.py @@ -186,7 +186,7 @@ QEMU_TEMPLATE_PROPERTIES = { "create_config_disk": { "description": "Automatically create a config disk on HDD disk interface (secondary slave)", "type": "boolean", - "default": True + "default": False }, "on_close": { "description": "Action to execute on the VM is closed", From 4843084158313a9c4ecfcd3ff4b5669647f1f48d Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 18 Aug 2020 10:54:11 +0930 Subject: [PATCH 15/17] Prioritize the config disk over HD-D for Qemu VMs. Fixes https://github.com/GNS3/gns3-gui/issues/3036 (cherry picked from commit c12b675691e01bedf093c57660db6a5ad19f39eb) --- gns3server/compute/qemu/qemu_vm.py | 37 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 62148042..611442e3 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1781,6 +1781,10 @@ class QemuVM(BaseNode): drives = ["a", "b", "c", "d"] for disk_index, drive in enumerate(drives): + # prioritize config disk over harddisk d + if drive == 'd' and self._create_config_disk: + continue + disk_image = getattr(self, "_hd{}_disk_image".format(drive)) if not disk_image: continue @@ -1846,24 +1850,21 @@ class QemuVM(BaseNode): # config disk disk_image = getattr(self, "config_disk_image") if disk_image and self._create_config_disk: - if getattr(self, "_hdd_disk_image"): - log.warning("Config disk: blocked by disk image 'hdd'") - else: - disk_name = getattr(self, "config_disk_name") - disk = os.path.join(self.working_dir, disk_name) - if self.hdd_disk_interface == "none": - # use the HDA interface type if none has been configured for HDD - self.hdd_disk_interface = getattr(self, "hda_disk_interface", "none") - await self._import_config() - disk_exists = os.path.exists(disk) - if not disk_exists: - try: - shutil.copyfile(disk_image, disk) - disk_exists = True - except OSError as e: - log.warning("Could not create '{}' disk image: {}".format(disk_name, e)) - if disk_exists: - options.extend(self._disk_interface_options(disk, 3, self.hdd_disk_interface, "raw")) + disk_name = getattr(self, "config_disk_name") + disk = os.path.join(self.working_dir, disk_name) + if self.hdd_disk_interface == "none": + # use the HDA interface type if none has been configured for HDD + self.hdd_disk_interface = getattr(self, "hda_disk_interface", "none") + await self._import_config() + disk_exists = os.path.exists(disk) + if not disk_exists: + try: + shutil.copyfile(disk_image, disk) + disk_exists = True + except OSError as e: + log.warning("Could not create '{}' disk image: {}".format(disk_name, e)) + if disk_exists: + options.extend(self._disk_interface_options(disk, 3, self.hdd_disk_interface, "raw")) return options From de2b9caeeb7770a97b8aa561ddd71f42df287aa3 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Mon, 19 Oct 2020 03:19:22 +0200 Subject: [PATCH 16/17] Use HDD disk image as startup QEMU config disk --- gns3server/compute/qemu/qemu_vm.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 611442e3..e421790a 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1722,11 +1722,21 @@ class QemuVM(BaseNode): async def _import_config(self): disk_name = getattr(self, "config_disk_name") + if not disk_name: + return + disk = os.path.join(self.working_dir, disk_name) zip_file = os.path.join(self.working_dir, "config.zip") - if not disk_name or not os.path.exists(zip_file): + startup_config = self.hdd_disk_image + if startup_config and startup_config.lower().endswith(".zip") and \ + not os.path.exists(zip_file) and not os.path.exists(disk): + try: + shutil.copyfile(startup_config, zip_file) + except OSError as e: + log.warning("Can't access startup config: {}".format(e)) + self.project.emit("log.warning", {"message": "{}: Can't access startup config: {}".format(self._name, e)}) + if not os.path.exists(zip_file): return config_dir = os.path.join(self.working_dir, "configs") - disk = os.path.join(self.working_dir, disk_name) disk_tmp = disk + ".tmp" try: os.mkdir(config_dir) From e45bc5aec15ab463af4d545a432e79297cf466b9 Mon Sep 17 00:00:00 2001 From: Bernhard Ehlers Date: Thu, 5 Nov 2020 15:00:44 +0100 Subject: [PATCH 17/17] Fix mcopy error messages --- gns3server/compute/qemu/qemu_vm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index e421790a..5b2ffe5d 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1678,9 +1678,9 @@ class QemuVM(BaseNode): mbr = img_file.read(512) part_type, offset, signature = struct.unpack("<450xB3xL52xH", mbr) if signature != 0xAA55: - raise OSError("mcopy failure: {}: invalid MBR".format(image)) + raise OSError("{}: invalid MBR".format(image)) if part_type not in (1, 4, 6, 11, 12, 14): - raise OSError("mcopy failure: {}: invalid partition type {:02X}" + raise OSError("{}: invalid partition type {:02X}" .format(image, part_type)) part_image = image + "@@{}S".format(offset)