From b408f297264ffe418438878fbef62b31f65cb67d Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 26 Apr 2016 13:12:42 -0600 Subject: [PATCH 01/11] Stricter checks to match VMware version to the right vmrun (VIX library) version. Also checks the VIX library version when only using the GNS3 VM running in VMware. --- gns3server/modules/vmware/__init__.py | 79 ++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/gns3server/modules/vmware/__init__.py b/gns3server/modules/vmware/__init__.py index 29150421..187f099c 100644 --- a/gns3server/modules/vmware/__init__.py +++ b/gns3server/modules/vmware/__init__.py @@ -108,7 +108,7 @@ class VMware(BaseManager): vmrun_path = shutil.which("vmrun") if not vmrun_path: - raise VMwareError("Could not find vmrun") + raise VMwareError("Could not find VMware vmrun, please make sure it is installed") if not os.path.isfile(vmrun_path): raise VMwareError("vmrun {} is not accessible".format(vmrun_path)) if not os.access(vmrun_path, os.X_OK): @@ -137,6 +137,50 @@ class VMware(BaseManager): version = match.group(1) return version + @asyncio.coroutine + def _check_vmware_player_requirements(self, player_version): + """ + Check minimum requirements to use VMware Player. + + VIX 1.13 was the release for Player 6. + VIX 1.14 was the release for Player 7. + VIX 1.15 was the release for Workstation Player 12. + + :param player_version: VMware Player major version. + """ + + player_version = int(player_version) + if player_version < 6: + raise VMwareError("Using VMware Player requires version 6 or above") + elif player_version == 6: + yield from self.check_vmrun_version(minimum_required_version="1.13") + elif player_version == 7: + yield from self.check_vmrun_version(minimum_required_version="1.14") + elif player_version >= 12: + yield from self.check_vmrun_version(minimum_required_version="1.15") + + @asyncio.coroutine + def _check_vmware_workstation_requirements(self, ws_version): + """ + Check minimum requirements to use VMware Workstation. + + VIX 1.13 was the release for Workstation 10. + VIX 1.14 was the release for Workstation 11. + VIX 1.15 was the release for Workstation Pro 12. + + :param ws_version: VMware Workstation major version. + """ + + ws_version = int(ws_version) + if ws_version < 10: + raise VMwareError("Using VMware Workstation requires version 10 or above") + elif ws_version == 10: + yield from self.check_vmrun_version(minimum_required_version="1.13") + elif ws_version == 11: + yield from self.check_vmrun_version(minimum_required_version="1.14") + elif ws_version >= 12: + yield from self.check_vmrun_version(minimum_required_version="1.15") + @asyncio.coroutine def check_vmware_version(self): """ @@ -150,15 +194,12 @@ class VMware(BaseManager): player_version = self._find_vmware_version_registry(r"SOFTWARE\Wow6432Node\VMware, Inc.\VMware Player") if player_version: log.debug("VMware Player version {} detected".format(player_version)) - if int(player_version) < 6: - raise VMwareError("Using VMware Player requires version 6 or above") + yield from self._check_vmware_player_requirements(player_version) else: log.warning("Could not find VMware version") else: log.debug("VMware Workstation version {} detected".format(ws_version)) - if int(ws_version) < 10: - raise VMwareError("Using VMware Workstation requires version 10 or above") - return + yield from self._check_vmware_workstation_requirements(ws_version) else: if sys.platform.startswith("darwin"): if not os.path.isdir("/Applications/VMware Fusion.app"): @@ -174,16 +215,16 @@ class VMware(BaseManager): match = re.search("VMware Workstation ([0-9]+)\.", output) version = None if match: + # VMware Workstation has been detected version = match.group(1) log.debug("VMware Workstation version {} detected".format(version)) - if int(version) < 10: - raise VMwareError("Using VMware Workstation requires version 10 or above") + yield from self._check_vmware_workstation_requirements(version) match = re.search("VMware Player ([0-9]+)\.", output) if match: + # VMware Player has been detected version = match.group(1) log.debug("VMware Player version {} detected".format(version)) - if int(version) < 6: - raise VMwareError("Using VMware Player requires version 6 or above") + yield from self._check_vmware_player_requirements(version) if version is None: log.warning("Could not find VMware version. Output of VMware: {}".format(output)) raise VMwareError("Could not find VMware version. Output of VMware: {}".format(output)) @@ -352,7 +393,17 @@ class VMware(BaseManager): return stdout_data.decode("utf-8", errors="ignore").splitlines() @asyncio.coroutine - def check_vmrun_version(self): + def check_vmrun_version(self, minimum_required_version="1.13"): + """ + Checks the vmrun version. + + VMware VIX library version must be at least >= 1.13 by default + VIX 1.13 was the release for VMware Fusion 6, Workstation 10, and Player 6. + VIX 1.14 was the release for VMware Fusion 7, Workstation 11 and Player 7. + VIX 1.15 was the release for VMware Fusion 8, Workstation Pro 12 and Workstation Player 12. + + :param required_version: required vmrun version number + """ with (yield from self._execute_lock): vmrun_path = self.vmrun_path @@ -366,9 +417,9 @@ class VMware(BaseManager): if match: version = match.group(1) log.debug("VMware vmrun version {} detected".format(version)) - if parse_version(version) < parse_version("1.13"): - # VMware VIX library version must be at least >= 1.13 - raise VMwareError("VMware vmrun executable version must be >= version 1.13") + if parse_version(version) < parse_version(minimum_required_version): + + raise VMwareError("VMware vmrun executable version must be >= version {}".format(minimum_required_version)) if version is None: log.warning("Could not find VMware vmrun version. Output: {}".format(output)) raise VMwareError("Could not find VMware vmrun version. Output: {}".format(output)) From 4a91d8a6a5518f3d88f0d09bdbf24cbc309dff32 Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 26 Apr 2016 15:06:22 -0600 Subject: [PATCH 02/11] Prevent non linked cloned hard disks to be detached when using VirtualBox linked cloned VMs. Fixes #1184. --- gns3server/modules/virtualbox/virtualbox_vm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gns3server/modules/virtualbox/virtualbox_vm.py b/gns3server/modules/virtualbox/virtualbox_vm.py index a3748414..7e5174d0 100644 --- a/gns3server/modules/virtualbox/virtualbox_vm.py +++ b/gns3server/modules/virtualbox/virtualbox_vm.py @@ -368,12 +368,12 @@ class VirtualBoxVM(BaseVM): hdd_files = yield from self._get_all_hdd_files() vm_info = yield from self._get_vm_info() for entry, value in vm_info.items(): - match = re.search("^([\s\w]+)\-(\d)\-(\d)$", entry) + match = re.search("^([\s\w]+)\-(\d)\-(\d)$", entry) # match Controller-PortNumber-DeviceNumber entry if match: controller = match.group(1) port = match.group(2) device = match.group(3) - if value in hdd_files: + if value in hdd_files and os.path.exists(os.path.join(self.working_dir, self._vmname, "Snapshots", os.path.basename(value))): log.info("VirtualBox VM '{name}' [{id}] detaching HDD {controller} {port} {device}".format(name=self.name, id=self.id, controller=controller, From 4fc5364ab55828d41e40def1d5f4586df0b01b97 Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 26 Apr 2016 15:44:11 -0600 Subject: [PATCH 03/11] More robust save/restore for VirtualBox linked clone VM hard disks. --- .../modules/virtualbox/virtualbox_vm.py | 128 ++++++++++++------ 1 file changed, 85 insertions(+), 43 deletions(-) diff --git a/gns3server/modules/virtualbox/virtualbox_vm.py b/gns3server/modules/virtualbox/virtualbox_vm.py index 7e5174d0..bbc262cc 100644 --- a/gns3server/modules/virtualbox/virtualbox_vm.py +++ b/gns3server/modules/virtualbox/virtualbox_vm.py @@ -159,7 +159,7 @@ class VirtualBoxVM(BaseVM): if self.id and os.path.isdir(os.path.join(self.working_dir, self._vmname)): vbox_file = os.path.join(self.working_dir, self._vmname, self._vmname + ".vbox") yield from self.manager.execute("registervm", [vbox_file]) - yield from self._reattach_hdds() + yield from self._reattach_linked_hdds() else: yield from self._create_linked_clone() @@ -314,7 +314,10 @@ class VirtualBoxVM(BaseVM): return hdds @asyncio.coroutine - def _reattach_hdds(self): + def _reattach_linked_hdds(self): + """ + Reattach linked cloned hard disks. + """ hdd_info_file = os.path.join(self.working_dir, self._vmname, "hdd_info.json") try: @@ -333,10 +336,67 @@ class VirtualBoxVM(BaseVM): device=hdd_info["device"], medium=hdd_file)) - yield from self._storage_attach('--storagectl "{}" --port {} --device {} --type hdd --medium "{}"'.format(hdd_info["controller"], - hdd_info["port"], - hdd_info["device"], - hdd_file)) + try: + yield from self._storage_attach('--storagectl "{}" --port {} --device {} --type hdd --medium "{}"'.format(hdd_info["controller"], + hdd_info["port"], + hdd_info["device"], + hdd_file)) + + except VirtualBoxError as e: + log.warn("VirtualBox VM '{name}' [{id}] error reattaching HDD {controller} {port} {device} {medium}: {error}".format(name=self.name, + id=self.id, + controller=hdd_info["controller"], + port=hdd_info["port"], + device=hdd_info["device"], + medium=hdd_file, + error=e)) + continue + + @asyncio.coroutine + def save_linked_hdds_info(self): + """ + Save linked cloned hard disks information. + + :returns: disk table information + """ + + hdd_table = [] + if self._linked_clone: + if os.path.exists(self.working_dir): + hdd_files = yield from self._get_all_hdd_files() + vm_info = yield from self._get_vm_info() + for entry, value in vm_info.items(): + match = re.search("^([\s\w]+)\-(\d)\-(\d)$", entry) # match Controller-PortNumber-DeviceNumber entry + if match: + controller = match.group(1) + port = match.group(2) + device = match.group(3) + if value in hdd_files and os.path.exists(os.path.join(self.working_dir, self._vmname, "Snapshots", os.path.basename(value))): + log.info("VirtualBox VM '{name}' [{id}] detaching HDD {controller} {port} {device}".format(name=self.name, + id=self.id, + controller=controller, + port=port, + device=device)) + hdd_table.append( + { + "hdd": os.path.basename(value), + "controller": controller, + "port": port, + "device": device, + } + ) + + if hdd_table: + try: + hdd_info_file = os.path.join(self.working_dir, self._vmname, "hdd_info.json") + with open(hdd_info_file, "w", encoding="utf-8") as f: + json.dump(hdd_table, f, indent=4) + except OSError as e: + log.warning("VirtualBox VM '{name}' [{id}] could not write HHD info file: {error}".format(name=self.name, + id=self.id, + error=e.strerror)) + + return hdd_table @asyncio.coroutine def close(self): @@ -363,47 +423,29 @@ class VirtualBoxVM(BaseVM): yield from self.stop() if self._linked_clone: - hdd_table = [] - if os.path.exists(self.working_dir): - hdd_files = yield from self._get_all_hdd_files() - vm_info = yield from self._get_vm_info() - for entry, value in vm_info.items(): - match = re.search("^([\s\w]+)\-(\d)\-(\d)$", entry) # match Controller-PortNumber-DeviceNumber entry - if match: - controller = match.group(1) - port = match.group(2) - device = match.group(3) - if value in hdd_files and os.path.exists(os.path.join(self.working_dir, self._vmname, "Snapshots", os.path.basename(value))): - log.info("VirtualBox VM '{name}' [{id}] detaching HDD {controller} {port} {device}".format(name=self.name, - id=self.id, - controller=controller, - port=port, - device=device)) - yield from self._storage_attach('--storagectl "{}" --port {} --device {} --type hdd --medium none'.format(controller, - port, - device)) - hdd_table.append( - { - "hdd": os.path.basename(value), - "controller": controller, - "port": port, - "device": device, - } - ) + hdd_table = yield from self.save_linked_hdds_info() + for hdd in hdd_table.copy(): + log.info("VirtualBox VM '{name}' [{id}] detaching HDD {controller} {port} {device}".format(name=self.name, + id=self.id, + controller=hdd["controller"], + port=hdd["port"], + device=hdd["device"])) + try: + yield from self._storage_attach('--storagectl "{}" --port {} --device {} --type hdd --medium none'.format(hdd["controller"], + hdd["port"], + hdd["device"])) + except VirtualBoxError as e: + log.warn("VirtualBox VM '{name}' [{id}] error detaching HDD {controller} {port} {device}: {error}".format(name=self.name, + id=self.id, + controller=hdd["controller"], + port=hdd["port"], + device=hdd["device"], + error=e)) + continue log.info("VirtualBox VM '{name}' [{id}] unregistering".format(name=self.name, id=self.id)) yield from self.manager.execute("unregistervm", [self._name]) - if hdd_table: - try: - hdd_info_file = os.path.join(self.working_dir, self._vmname, "hdd_info.json") - with open(hdd_info_file, "w", encoding="utf-8") as f: - json.dump(hdd_table, f, indent=4) - except OSError as e: - log.warning("VirtualBox VM '{name}' [{id}] could not write HHD info file: {error}".format(name=self.name, - id=self.id, - error=e.strerror)) - log.info("VirtualBox VM '{name}' [{id}] closed".format(name=self.name, id=self.id)) self._closed = True From cf1b7ebe1f8b0e0fab21423239d84c8defd515b6 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Wed, 27 Apr 2016 15:12:15 +0200 Subject: [PATCH 04/11] Raise error if qemu can not create backing image --- gns3server/modules/qemu/qemu_vm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gns3server/modules/qemu/qemu_vm.py b/gns3server/modules/qemu/qemu_vm.py index 280b7a0f..1992c30f 100644 --- a/gns3server/modules/qemu/qemu_vm.py +++ b/gns3server/modules/qemu/qemu_vm.py @@ -1261,6 +1261,8 @@ class QemuVM(BaseVM): "backing_file={}".format(disk_image), "-f", "qcow2", disk) retcode = yield from process.wait() + if retcode is not None and retcode != 0: + raise QemuError("Could not create {} disk image".format(disk_name)) log.info("{} returned with {}".format(qemu_img_path, retcode)) except (OSError, subprocess.SubprocessError) as e: raise QemuError("Could not create {} disk image {}".format(disk_name, e)) From 85f571ee8bffb05f63cd922d394562c209ceb2b1 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Thu, 28 Apr 2016 16:35:18 +0200 Subject: [PATCH 05/11] Solve wintypes import --- gns3server/modules/vmware/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gns3server/modules/vmware/__init__.py b/gns3server/modules/vmware/__init__.py index 187f099c..8825d853 100644 --- a/gns3server/modules/vmware/__init__.py +++ b/gns3server/modules/vmware/__init__.py @@ -646,6 +646,7 @@ class VMware(BaseManager): if sys.platform.startswith("win"): import ctypes + import ctypes.wintypes path = ctypes.create_unicode_buffer(ctypes.wintypes.MAX_PATH) ctypes.windll.shell32.SHGetFolderPathW(None, 5, None, 0, path) documents_folder = path.value From 8a19afd618955356f7b429406cf613d0a62b39a6 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Thu, 28 Apr 2016 18:27:33 +0200 Subject: [PATCH 06/11] 1.4.6 --- CHANGELOG | 11 +++++++++++ gns3server/version.py | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 31faa0b0..d99410fb 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,16 @@ # Change Log +## 1.4.6 28/04/2016 + +* More robust save/restore for VirtualBox linked clone VM hard disks. +* Prevent non linked cloned hard disks to be detached when using VirtualBox linked cloned VMs. Fixes #1184. +* Stricter checks to match VMware version to the right vmrun (VIX library) version. Also checks the VIX library version when only using the GNS3 VM running in VMware. +* Allow only .pcap to be downloaded from remote stream API +* Fix incrementation of qemu mac address +* Clear warnings about using linked clones with VMware Player. +* Alternative method to find the Documents folder on Windows. +* Add IOU support and install config in /etc + ## 1.4.5 23/03/2016 * Stop the VMware VM if there is an error while setting up the network connections or console. diff --git a/gns3server/version.py b/gns3server/version.py index aaced8f2..23385d90 100644 --- a/gns3server/version.py +++ b/gns3server/version.py @@ -23,5 +23,5 @@ # or negative for a release candidate or beta (after the base version # number has been incremented) -__version__ = "1.4.6dev1" -__version_info__ = (1, 4, 6, -99) +__version__ = "1.4.6" +__version_info__ = (1, 4, 6, 0) From 848120c354b8e827015376c26103e1d68614fb68 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Thu, 28 Apr 2016 18:38:11 +0200 Subject: [PATCH 07/11] 1.4.7dev1 --- gns3server/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gns3server/version.py b/gns3server/version.py index 23385d90..0cc81f25 100644 --- a/gns3server/version.py +++ b/gns3server/version.py @@ -23,5 +23,5 @@ # or negative for a release candidate or beta (after the base version # number has been incremented) -__version__ = "1.4.6" -__version_info__ = (1, 4, 6, 0) +__version__ = "1.4.7dev1" +__version_info__ = (1, 4, 7, -99) From b92e64e5077b54b3a1d44b4b5e3b56898817b407 Mon Sep 17 00:00:00 2001 From: grossmj Date: Thu, 28 Apr 2016 16:21:14 -0600 Subject: [PATCH 08/11] pywin32 is not updated on pypi. --- setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/setup.py b/setup.py index 55a29001..d0c8d0d9 100644 --- a/setup.py +++ b/setup.py @@ -40,9 +40,6 @@ class PyTest(TestCommand): dependencies = open("requirements.txt", "r").read().splitlines() -if sys.platform.startswith("win"): - dependencies.append("pywin32>=219") - setup( name="gns3-server", version=__import__("gns3server").__version__, From 5c1522f24a3c34629daf35dbe278483f4f34cedd Mon Sep 17 00:00:00 2001 From: grossmj Date: Thu, 28 Apr 2016 23:47:17 -0600 Subject: [PATCH 09/11] Bump version to 1.5.0dev2 --- gns3server/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gns3server/version.py b/gns3server/version.py index e99a4a97..8ed6cebe 100644 --- a/gns3server/version.py +++ b/gns3server/version.py @@ -23,5 +23,5 @@ # or negative for a release candidate or beta (after the base version # number has been incremented) -__version__ = "1.5.0dev1" +__version__ = "1.5.0dev2" __version_info__ = (1, 5, 0, -99) From 936faaba5a010f88e34abc5f22c102a0537bc3d2 Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Mon, 2 May 2016 10:30:21 +0200 Subject: [PATCH 10/11] gns3z => gns3project --- gns3server/handlers/api/project_handler.py | 4 ++-- tests/handlers/api/test_project.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gns3server/handlers/api/project_handler.py b/gns3server/handlers/api/project_handler.py index ea9f4132..3c4a0fa2 100644 --- a/gns3server/handlers/api/project_handler.py +++ b/gns3server/handlers/api/project_handler.py @@ -361,8 +361,8 @@ class ProjectHandler: pm = ProjectManager.instance() project = pm.get_project(request.match_info["project_id"]) - response.content_type = 'application/gns3z' - response.headers['CONTENT-DISPOSITION'] = 'attachment; filename="{}.gns3z"'.format(project.name) + response.content_type = 'application/gns3project' + response.headers['CONTENT-DISPOSITION'] = 'attachment; filename="{}.gns3project"'.format(project.name) response.enable_chunked_encoding() # Very important: do not send a content length otherwise QT close the connection but curl can consume the Feed response.content_length = None diff --git a/tests/handlers/api/test_project.py b/tests/handlers/api/test_project.py index cf952794..90a4a554 100644 --- a/tests/handlers/api/test_project.py +++ b/tests/handlers/api/test_project.py @@ -294,8 +294,8 @@ def test_export(server, tmpdir, loop, project): response = server.get("/projects/{project_id}/export".format(project_id=project.id), raw=True) assert response.status == 200 - assert response.headers['CONTENT-TYPE'] == 'application/gns3z' - assert response.headers['CONTENT-DISPOSITION'] == 'attachment; filename="{}.gns3z"'.format(project.name) + assert response.headers['CONTENT-TYPE'] == 'application/gns3project' + assert response.headers['CONTENT-DISPOSITION'] == 'attachment; filename="{}.gns3project"'.format(project.name) with open(str(tmpdir / 'project.zip'), 'wb+') as f: f.write(response.body) From c17e00204fb478d9535a2c194d35a0d92ce7609a Mon Sep 17 00:00:00 2001 From: Julien Duponchelle Date: Mon, 2 May 2016 16:55:05 +0200 Subject: [PATCH 11/11] Auto reconnect to docker if connection was closed Fix #507 --- gns3server/modules/docker/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gns3server/modules/docker/__init__.py b/gns3server/modules/docker/__init__.py index 22f74b61..3442d3fa 100644 --- a/gns3server/modules/docker/__init__.py +++ b/gns3server/modules/docker/__init__.py @@ -49,7 +49,7 @@ class Docker(BaseManager): @asyncio.coroutine def connector(self): - if not self._connected: + if not self._connected or self._connector.closed: try: self._connector = aiohttp.connector.UnixConnector(self._server_url) self._connected = True