diff --git a/CHANGELOG b/CHANGELOG index 6ad8a76a..a234b157 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -140,6 +140,18 @@ * Implement #1153 into 2.2 branch. * Pin prompt-toolkit to latest version 1.0.15 +## 2.1.12 23/01/2019 + +* Tune how to get the size of SVG images. Ref https://github.com/GNS3/gns3-gui/issues/2674. +* Automatically create a symbolic link to the IOU image in the IOU working directory. Fixes #1484 +* Fix link pause/filters only work for the first interface of Docker containers. Fixes #1482 +* Telnet console resize support for Docker VM. +* Fix _fix_permissions() garbles permissions in Docker VM. Ref #1428 +* Fix "None is not of type 'integer'" when opening project containing a Qemu VM. Fixes #2610. +* Only require Xtigervnc or Xvfb+x11vnc for Docker with vnc console. Ref #1438 +* Support tigervnc in Docker VM. Ref #1438 +* Update minimum VIX version requirements for VMware. Ref #1415. + ## 2.1.11 28/09/2018 * Catch some exceptions. diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index cccab216..fcf26e7e 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -279,7 +279,7 @@ class BaseManager: destination_dir = destination_node.working_dir try: shutil.rmtree(destination_dir) - shutil.copytree(source_node.working_dir, destination_dir) + shutil.copytree(source_node.working_dir, destination_dir, symlinks=True, ignore_dangling_symlinks=True) except OSError as e: raise aiohttp.web.HTTPConflict(text="Cannot duplicate node data: {}".format(e)) diff --git a/gns3server/compute/docker/__init__.py b/gns3server/compute/docker/__init__.py index 37bc0d80..b24be8d2 100644 --- a/gns3server/compute/docker/__init__.py +++ b/gns3server/compute/docker/__init__.py @@ -198,7 +198,10 @@ class Docker(BaseManager): if progress_callback: progress_callback("Pulling '{}' from docker hub".format(image)) - response = await self.http_query("POST", "images/create", params={"fromImage": image}, timeout=None) + try: + response = await self.http_query("POST", "images/create", params={"fromImage": image}, timeout=None) + except DockerError as e: + raise DockerError("Could not pull the '{}' image from Docker Hub, please check your Internet connection (original error: {})".format(image, e)) # The pull api will stream status via an HTTP JSON stream content = "" while True: diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index e73dc8b4..47a0b34e 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -353,6 +353,9 @@ class DockerVM(BaseNode): if self._environment: for e in self._environment.strip().split("\n"): e = e.strip() + if e.split("=")[0] == "": + self.project.emit("log.warning", {"message": "{} has invalid environment variable: {}".format(self.name, e)}) + continue if not e.startswith("GNS3_"): formatted = self._format_env(variables, e) params["Env"].append(formatted) diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index b615a5f3..08a43e54 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -535,8 +535,9 @@ class IOUVM(BaseNode): # on newer images, see https://github.com/GNS3/gns3-server/issues/1484 try: symlink = os.path.join(self.working_dir, os.path.basename(self.path)) - if not os.path.islink(symlink): - os.symlink(self.path, symlink) + if os.path.islink(symlink): + os.unlink(symlink) + os.symlink(self.path, symlink) except OSError as e: raise IOUError("Could not create symbolic link: {}".format(e)) diff --git a/gns3server/compute/qemu/__init__.py b/gns3server/compute/qemu/__init__.py index 6cae4a00..cf37224b 100644 --- a/gns3server/compute/qemu/__init__.py +++ b/gns3server/compute/qemu/__init__.py @@ -185,8 +185,8 @@ class Qemu(BaseManager): return "" else: try: - output = await subprocess_check_output(qemu_path, "-version") - match = re.search(r"version\s+([0-9a-z\-\.]+)", output) + output = await subprocess_check_output(qemu_path, "-version", "-nographic") + match = re.search("version\s+([0-9a-z\-\.]+)", output) if match: version = match.group(1) return version diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 7d8ee21e..19a19241 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1736,18 +1736,18 @@ class QemuVM(BaseNode): return network_options - def _graphic(self): + async def _disable_graphics(self): """ - Adds the correct graphic options depending of the OS + Disable graphics depending of the QEMU version """ - if sys.platform.startswith("win"): - return [] - if len(os.environ.get("DISPLAY", "")) > 0: + if any(opt in self._options for opt in ["-display", "-nographic", "-curses", "-sdl" "-spice", "-vnc"]): return [] - if "-nographic" not in self._options: + version = await self.manager.get_qemu_version(self.qemu_path) + if version and parse_version(version) >= parse_version("3.0"): + return ["-display", "none"] + else: return ["-nographic"] - return [] async def _run_with_hardware_acceleration(self, qemu_path, options): """ @@ -1920,12 +1920,12 @@ class QemuVM(BaseNode): raise QemuError("Console type {} is unknown".format(self._console_type)) command.extend(self._monitor_options()) command.extend((await self._network_options())) - command.extend(self._graphic()) if self.on_close != "save_vm_state": await self._clear_save_vm_stated() else: command.extend((await self._saved_state_option())) - + if self._console_type == "telnet": + command.extend((await self._disable_graphics())) if additional_options: try: command.extend(shlex.split(additional_options)) diff --git a/gns3server/controller/drawing.py b/gns3server/controller/drawing.py index e20573f8..c730cd80 100644 --- a/gns3server/controller/drawing.py +++ b/gns3server/controller/drawing.py @@ -55,7 +55,7 @@ class Drawing: return self._id @property - def ressource_filename(self): + def resource_filename(self): """ If the svg content has been dump to an external file return is name otherwise None """ diff --git a/gns3server/controller/export_project.py b/gns3server/controller/export_project.py index 17919286..49098c10 100644 --- a/gns3server/controller/export_project.py +++ b/gns3server/controller/export_project.py @@ -30,7 +30,7 @@ import logging log = logging.getLogger(__name__) -async def export_project(project, temporary_dir, include_images=False, keep_compute_id=False, allow_all_nodes=False): +async def export_project(project, temporary_dir, include_images=False, keep_compute_id=False, allow_all_nodes=False, reset_mac_addresses=False): """ Export a project to a zip file. @@ -41,6 +41,7 @@ async def export_project(project, temporary_dir, include_images=False, keep_comp :param include images: save OS images to the zip file :param keep_compute_id: If false replace all compute id by local (standard behavior for .gns3project to make it portable) :param allow_all_nodes: Allow all nodes type to be include in the zip even if not portable + :param reset_mac_addresses: Reset MAC addresses for every nodes. :returns: ZipStream object """ @@ -60,10 +61,10 @@ async def export_project(project, temporary_dir, include_images=False, keep_comp # First we process the .gns3 in order to be sure we don't have an error for file in os.listdir(project._path): if file.endswith(".gns3"): - await _patch_project_file(project, os.path.join(project._path, file), zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir) + await _patch_project_file(project, os.path.join(project._path, file), zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir, reset_mac_addresses) # Export the local files - for root, dirs, files in os.walk(project._path, topdown=True): + for root, dirs, files in os.walk(project._path, topdown=True, followlinks=False): files = [f for f in files if _is_exportable(os.path.join(root, f))] for file in files: path = os.path.join(root, file) @@ -124,6 +125,7 @@ def _patch_mtime(path): new_mtime = file_date.replace(year=1980).timestamp() os.utime(path, (st.st_atime, new_mtime)) + def _is_exportable(path): """ :returns: True if file should not be included in the final archive @@ -133,6 +135,10 @@ def _is_exportable(path): if path.endswith("snapshots"): return False + # do not export symlinks + if os.path.islink(path): + return False + # do not export directories of snapshots if "{sep}snapshots{sep}".format(sep=os.path.sep) in path: return False @@ -153,7 +159,7 @@ def _is_exportable(path): return True -async def _patch_project_file(project, path, zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir): +async def _patch_project_file(project, path, zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir, reset_mac_addresses): """ Patch a project file (.gns3) to export a project. The .gns3 file is renamed to project.gns3 @@ -186,6 +192,10 @@ async def _patch_project_file(project, path, zstream, include_images, keep_compu if "properties" in node and node["node_type"] != "docker": for prop, value in node["properties"].items(): + # reset the MAC address + if reset_mac_addresses and prop in ("mac_addr", "mac_address"): + node["properties"][prop] = None + if node["node_type"] == "iou": if not prop == "path": continue diff --git a/gns3server/controller/gns3vm/vmware_gns3_vm.py b/gns3server/controller/gns3vm/vmware_gns3_vm.py index 9976000a..cdc3cff4 100644 --- a/gns3server/controller/gns3vm/vmware_gns3_vm.py +++ b/gns3server/controller/gns3vm/vmware_gns3_vm.py @@ -69,13 +69,15 @@ class VMwareGNS3VM(BaseGNS3VM): if ram % 4 != 0: raise GNS3VMError("Allocated memory {} for the GNS3 VM must be a multiple of 4".format(ram)) - available_vcpus = psutil.cpu_count(logical=False) + available_vcpus = psutil.cpu_count() if vcpus > available_vcpus: raise GNS3VMError("You have allocated too many vCPUs for the GNS3 VM! (max available is {} vCPUs)".format(available_vcpus)) + cores_per_sockets = int(available_vcpus / psutil.cpu_count(logical=False)) try: pairs = VMware.parse_vmware_file(self._vmx_path) pairs["numvcpus"] = str(vcpus) + pairs["cpuid.coresPerSocket"] = str(cores_per_sockets) pairs["memsize"] = str(ram) VMware.write_vmx_file(self._vmx_path, pairs) log.info("GNS3 VM vCPU count set to {} and RAM amount set to {}".format(vcpus, ram)) diff --git a/gns3server/controller/import_project.py b/gns3server/controller/import_project.py index 54763136..771d0236 100644 --- a/gns3server/controller/import_project.py +++ b/gns3server/controller/import_project.py @@ -182,9 +182,11 @@ async def _move_files_to_compute(compute, project_id, directory, files_path): location = os.path.join(directory, files_path) if os.path.exists(location): - for (dirpath, dirnames, filenames) in os.walk(location): + for (dirpath, dirnames, filenames) in os.walk(location, followlinks=False): for filename in filenames: path = os.path.join(dirpath, filename) + if os.path.islink(path): + continue dst = os.path.relpath(path, directory) await _upload_file(compute, project_id, path, dst) await wait_run_in_executor(shutil.rmtree, os.path.join(directory, files_path)) @@ -210,9 +212,11 @@ def _import_images(controller, path): image_dir = controller.images_path() root = os.path.join(path, "images") - for (dirpath, dirnames, filenames) in os.walk(root): + for (dirpath, dirnames, filenames) in os.walk(root, followlinks=False): for filename in filenames: path = os.path.join(dirpath, filename) + if os.path.islink(path): + continue dst = os.path.join(image_dir, os.path.relpath(path, root)) os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.move(path, dst) diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 0ef03be5..a8384324 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -740,25 +740,27 @@ class Project: # We don't care if a compute is down at this step except (ComputeError, aiohttp.web.HTTPError, aiohttp.ClientResponseError, TimeoutError): pass - self._cleanPictures() + self._clean_pictures() self._status = "closed" if not ignore_notification: self.controller.notification.project_emit("project.closed", self.__json__()) self.reset() - def _cleanPictures(self): + def _clean_pictures(self): """ - Delete unused images + Delete unused pictures. """ - # Project have been deleted - if not os.path.exists(self.path): + # Project have been deleted or is loading or is not opened + if not os.path.exists(self.path) or self._loading or self._status != "opened": return try: pictures = set(os.listdir(self.pictures_directory)) for drawing in self._drawings.values(): try: - pictures.remove(drawing.ressource_filename) + resource_filename = drawing.resource_filename + if resource_filename: + pictures.remove(resource_filename) except KeyError: pass @@ -770,10 +772,12 @@ class Project: except KeyError: pass - for pict in pictures: - os.remove(os.path.join(self.pictures_directory, pict)) + for pic_filename in pictures: + path = os.path.join(self.pictures_directory, pic_filename) + log.info("Deleting unused picture '{}'".format(path)) + os.remove(path) except OSError as e: - log.warning(str(e)) + log.warning("Could not delete unused pictures: {}".format(e)) async def delete(self): @@ -962,7 +966,7 @@ class Project: assert self._status != "closed" try: with tempfile.TemporaryDirectory() as tmpdir: - zipstream = await export_project(self, tmpdir, keep_compute_id=True, allow_all_nodes=True) + zipstream = await export_project(self, tmpdir, keep_compute_id=True, allow_all_nodes=True, reset_mac_addresses=True) project_path = os.path.join(tmpdir, "project.gns3p") await wait_run_in_executor(self._create_duplicate_project_file, project_path, zipstream) with open(project_path, "rb") as f: diff --git a/gns3server/handlers/api/controller/template_handler.py b/gns3server/handlers/api/controller/template_handler.py index 00d93be6..6d6005d2 100644 --- a/gns3server/handlers/api/controller/template_handler.py +++ b/gns3server/handlers/api/controller/template_handler.py @@ -164,8 +164,9 @@ class TemplateHandler: controller = Controller.instance() project = controller.get_project(request.match_info["project_id"]) - await project.add_node_from_template(request.match_info["template_id"], - x=request.json["x"], - y=request.json["y"], - compute_id=request.json.get("compute_id")) + node = await project.add_node_from_template(request.match_info["template_id"], + x=request.json["x"], + y=request.json["y"], + compute_id=request.json.get("compute_id")) response.set_status(201) + response.json(node) diff --git a/gns3server/schemas/dynamips_vm.py b/gns3server/schemas/dynamips_vm.py index 992e1f48..1fc939a6 100644 --- a/gns3server/schemas/dynamips_vm.py +++ b/gns3server/schemas/dynamips_vm.py @@ -180,7 +180,7 @@ VM_CREATE_SCHEMA = { }, "mac_addr": { "description": "Base MAC address", - "type": "string", + "type": ["null", "string"], "minLength": 1, "pattern": "^([0-9a-fA-F]{4}\\.){2}[0-9a-fA-F]{4}$" }, @@ -402,7 +402,7 @@ VM_UPDATE_SCHEMA = { }, "mac_addr": { "description": "Base MAC address", - "type": "string", + "type": ["null", "string"], "minLength": 1, "pattern": "^([0-9a-fA-F]{4}\\.){2}[0-9a-fA-F]{4}$" }, @@ -646,7 +646,7 @@ VM_OBJECT_SCHEMA = { }, "mac_addr": { "description": "Base MAC address", - "type": "string", + "type": ["null", "string"] #"minLength": 1, #"pattern": "^([0-9a-fA-F]{4}\\.){2}[0-9a-fA-F]{4}$" }, diff --git a/gns3server/ubridge/ubridge_hypervisor.py b/gns3server/ubridge/ubridge_hypervisor.py index 0bc8884e..4fcf1d1f 100644 --- a/gns3server/ubridge/ubridge_hypervisor.py +++ b/gns3server/ubridge/ubridge_hypervisor.py @@ -68,7 +68,7 @@ class UBridgeHypervisor: connection_success = False last_exception = None while time.time() - begin < timeout: - await asyncio.sleep(0.01) + await asyncio.sleep(0.1) try: self._reader, self._writer = await asyncio.open_connection(host, self._port) except OSError as e: @@ -83,6 +83,7 @@ class UBridgeHypervisor: log.info("Connected to uBridge hypervisor on {}:{} after {:.4f} seconds".format(host, self._port, time.time() - begin)) try: + await asyncio.sleep(0.1) version = await self.send("hypervisor version") self._version = version[0].split("-", 1)[0] except IndexError: @@ -232,7 +233,7 @@ class UBridgeHypervisor: .format(host=self._host, port=self._port, command=command, run=self.is_running())) else: retries += 1 - await asyncio.sleep(0.1) + await asyncio.sleep(0.5) continue retries = 0 buf += chunk.decode("utf-8") diff --git a/tests/compute/qemu/test_qemu_vm.py b/tests/compute/qemu/test_qemu_vm.py index 9b156d6c..9ef16fe0 100644 --- a/tests/compute/qemu/test_qemu_vm.py +++ b/tests/compute/qemu/test_qemu_vm.py @@ -132,6 +132,7 @@ def test_is_running(vm, running_subprocess_mock): def test_start(loop, vm, running_subprocess_mock): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") with asyncio_patch("gns3server.compute.qemu.QemuVM.start_wrap_console"): with asyncio_patch("asyncio.create_subprocess_exec", return_value=running_subprocess_mock) as mock: loop.run_until_complete(asyncio.ensure_future(vm.start())) @@ -146,6 +147,7 @@ def test_stop(loop, vm, running_subprocess_mock): future = asyncio.Future() future.set_result(True) process.wait.return_value = future + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") with asyncio_patch("gns3server.compute.qemu.QemuVM.start_wrap_console"): with asyncio_patch("asyncio.create_subprocess_exec", return_value=process): @@ -229,6 +231,7 @@ def test_port_remove_nio_binding(vm, loop): def test_close(vm, port_manager, loop): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") with asyncio_patch("gns3server.compute.qemu.QemuVM.start_wrap_console"): with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()): loop.run_until_complete(asyncio.ensure_future(vm.start())) @@ -354,6 +357,7 @@ def test_disk_options(vm, tmpdir, loop, fake_qemu_img_binary): def test_cdrom_option(vm, tmpdir, loop, fake_qemu_img_binary): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") vm._cdrom_image = str(tmpdir / "test.iso") open(vm._cdrom_image, "w+").close() @@ -364,6 +368,7 @@ def test_cdrom_option(vm, tmpdir, loop, fake_qemu_img_binary): def test_bios_option(vm, tmpdir, loop, fake_qemu_img_binary): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") vm._bios_image = str(tmpdir / "test.img") open(vm._bios_image, "w+").close() @@ -470,6 +475,7 @@ def test_control_vm_expect_text(vm, loop, running_subprocess_mock): def test_build_command(vm, loop, fake_qemu_binary, port_manager): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") os.environ["DISPLAY"] = "0:0" with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: cmd = loop.run_until_complete(asyncio.ensure_future(vm._build_command())) @@ -493,7 +499,9 @@ def test_build_command(vm, loop, fake_qemu_binary, port_manager): "-device", "e1000,mac={},netdev=gns3-0".format(vm._mac_address), "-netdev", - "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport) + "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport), + "-display", + "none" ] @@ -502,6 +510,7 @@ def test_build_command_manual_uuid(vm, loop, fake_qemu_binary, port_manager): If user has set a uuid we keep it """ + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") vm.options = "-uuid e1c307a4-896f-11e6-81a5-3c07547807cc" os.environ["DISPLAY"] = "0:0" with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: @@ -541,7 +550,8 @@ def test_build_command_kvm(linux_platform, vm, loop, fake_qemu_binary, port_mana "-device", "e1000,mac={},netdev=gns3-0".format(vm._mac_address), "-netdev", - "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport) + "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport), + "-nographic" ] @@ -578,13 +588,15 @@ def test_build_command_kvm_2_4(linux_platform, vm, loop, fake_qemu_binary, port_ "-device", "e1000,mac={},netdev=gns3-0".format(vm._mac_address), "-netdev", - "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport) + "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport), + "-nographic" ] @pytest.mark.skipif(sys.platform.startswith("win"), reason="Not supported on Windows") def test_build_command_without_display(vm, loop, fake_qemu_binary): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="2.5.0") os.environ["DISPLAY"] = "" with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: cmd = loop.run_until_complete(asyncio.ensure_future(vm._build_command())) @@ -593,6 +605,7 @@ def test_build_command_without_display(vm, loop, fake_qemu_binary): def test_build_command_two_adapters(vm, loop, fake_qemu_binary, port_manager): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="2.5.0") os.environ["DISPLAY"] = "0:0" vm.adapters = 2 with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: @@ -622,7 +635,8 @@ def test_build_command_two_adapters(vm, loop, fake_qemu_binary, port_manager): "-device", "e1000,mac={},netdev=gns3-1".format(int_to_macaddress(macaddress_to_int(vm._mac_address) + 1)), "-netdev", - "socket,id=gns3-1,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio2.rport, nio2.lport) + "socket,id=gns3-1,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio2.rport, nio2.lport), + "-nographic" ] @@ -631,6 +645,7 @@ def test_build_command_two_adapters_mac_address(vm, loop, fake_qemu_binary, port Should support multiple base vmac address """ + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="2.5.0") vm.adapters = 2 vm.mac_address = "00:00:ab:0e:0f:09" mac_0 = vm._mac_address