From 5f932fee9fefc68fcbb1437586cd9e4d8d5800be Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 21 Jan 2019 16:01:03 +0700 Subject: [PATCH 01/17] Tune how to get the size of SVG images. Ref https://github.com/GNS3/gns3-gui/issues/2674. * Default for missing height/width is "100%" as defined in the SVG specification * Better error message, if viewBox attribute is missing * Removal of "%" in percent more fault tolerant by using rstrip("%") (cherry picked from commit e3757a895569d13cb57ead296fdea333d1eeb3e2) --- gns3server/controller/node.py | 2 +- .../handlers/api/controller/symbol_handler.py | 3 +- gns3server/utils/picture.py | 35 +++++++++++++------ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/gns3server/controller/node.py b/gns3server/controller/node.py index d42b6e68..796c08aa 100644 --- a/gns3server/controller/node.py +++ b/gns3server/controller/node.py @@ -254,7 +254,7 @@ class Node: try: self._width, self._height, filetype = self._project.controller.symbols.get_size(val) except (ValueError, OSError) as e: - log.error("Could not write symbol: {}".format(e)) + log.error("Could not set symbol: {}".format(e)) # If symbol is invalid we replace it by the default self.symbol = ":/symbols/computer.svg" if self._label is None: diff --git a/gns3server/handlers/api/controller/symbol_handler.py b/gns3server/handlers/api/controller/symbol_handler.py index 3d378766..92e2d12a 100644 --- a/gns3server/handlers/api/controller/symbol_handler.py +++ b/gns3server/handlers/api/controller/symbol_handler.py @@ -60,7 +60,7 @@ class SymbolHandler: r"/symbols/{symbol_id:.+}/raw", description="Write the symbol file", status_codes={ - 200: "Symbol returned" + 200: "Symbol written" }, raw=True) def upload(request, response): @@ -78,6 +78,7 @@ class SymbolHandler: f.write(chunk) except (UnicodeEncodeError, OSError) as e: raise aiohttp.web.HTTPConflict(text="Could not write symbol file '{}': {}".format(path, e)) + # Reset the symbol list controller.symbols.list() response.set_status(204) diff --git a/gns3server/utils/picture.py b/gns3server/utils/picture.py index d7c3103b..650bc6ba 100644 --- a/gns3server/utils/picture.py +++ b/gns3server/utils/picture.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import re import io import struct from xml.etree.ElementTree import ElementTree, ParseError @@ -103,25 +104,34 @@ def get_size(data, default_width=0, default_height=0): root = tree.getroot() try: - width_attr = root.attrib.get("width", "0") - height_attr = root.attrib.get("height", "0") + width_attr = root.attrib.get("width", "100%") + height_attr = root.attrib.get("height", "100%") if width_attr.endswith("%") or height_attr.endswith("%"): # check to viewBox attribute if width or height value is a percentage - _, _, width_attr, height_attr = root.attrib.get("viewBox").split() - else: - width = _svg_convert_size(width_attr) - height = _svg_convert_size(height_attr) + viewbox = root.attrib.get("viewBox") + if not viewbox: + raise ValueError("Invalid SVG file: missing viewBox attribute") + _, _, viewbox_width, viewbox_height = re.split(r'[\s,]+', viewbox) + if width_attr.endswith("%"): + width = _svg_convert_size(viewbox_width, width_attr) + else: + width = _svg_convert_size(width_attr) + if height_attr.endswith("%"): + height = _svg_convert_size(viewbox_height, height_attr) + else: + height = _svg_convert_size(height_attr) except (AttributeError, IndexError) as e: raise ValueError("Invalid SVG file: {}".format(e)) return width, height, filetype -def _svg_convert_size(size): +def _svg_convert_size(size, percent=None): """ Convert svg size to the px version :param size: String with the size + :param percent: String with the percentage, None = 100% """ # https://www.w3.org/TR/SVG/coords.html#Units @@ -133,8 +143,11 @@ def _svg_convert_size(size): "in": 90, "px": 1 } - if len(size) > 3: + factor = 1.0 + if len(size) >= 3: if size[-2:] in conversion_table: - return round(float(size[:-2]) * conversion_table[size[-2:]]) - - return round(float(size)) + factor = conversion_table[size[-2:]] + size = size[:-2] + if percent: + factor *= float(percent.rstrip("%")) / 100.0 + return round(float(size) * factor) From 3e21f96bf9c0b3d47fe763a00b26cf2bddbc74e3 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 21 Jan 2019 16:24:23 +0700 Subject: [PATCH 02/17] Fix indentation issue. Ref https://github.com/GNS3/gns3-gui/issues/2674 (cherry picked from commit c14d79a3d530b80ba94706e886af817862c00ff8) --- gns3server/utils/picture.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gns3server/utils/picture.py b/gns3server/utils/picture.py index 650bc6ba..3e9fd416 100644 --- a/gns3server/utils/picture.py +++ b/gns3server/utils/picture.py @@ -112,14 +112,14 @@ def get_size(data, default_width=0, default_height=0): if not viewbox: raise ValueError("Invalid SVG file: missing viewBox attribute") _, _, viewbox_width, viewbox_height = re.split(r'[\s,]+', viewbox) - if width_attr.endswith("%"): - width = _svg_convert_size(viewbox_width, width_attr) - else: - width = _svg_convert_size(width_attr) - if height_attr.endswith("%"): - height = _svg_convert_size(viewbox_height, height_attr) - else: - height = _svg_convert_size(height_attr) + if width_attr.endswith("%"): + width = _svg_convert_size(viewbox_width, width_attr) + else: + width = _svg_convert_size(width_attr) + if height_attr.endswith("%"): + height = _svg_convert_size(viewbox_height, height_attr) + else: + height = _svg_convert_size(height_attr) except (AttributeError, IndexError) as e: raise ValueError("Invalid SVG file: {}".format(e)) From bccdfc97d1d9aa91f19ec8003b4d330f3aecfd14 Mon Sep 17 00:00:00 2001 From: grossmj Date: Wed, 23 Jan 2019 15:40:38 +0800 Subject: [PATCH 03/17] Release 2.1.12 --- CHANGELOG | 12 ++++++++++++ gns3server/crash_report.py | 2 +- gns3server/version.py | 4 ++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index a5726009..bc8d5085 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,17 @@ # Change Log +## 2.1.12 23/01/2019 + +* Tune how to get the size of SVG images. Ref https://github.com/GNS3/gns3-gui/issues/2674. +* Automatically create a symbolic link to the IOU image in the IOU working directory. Fixes #1484 +* Fix link pause/filters only work for the first interface of Docker containers. Fixes #1482 +* Telnet console resize support for Docker VM. +* Fix _fix_permissions() garbles permissions in Docker VM. Ref #1428 +* Fix "None is not of type 'integer'" when opening project containing a Qemu VM. Fixes #2610. +* Only require Xtigervnc or Xvfb+x11vnc for Docker with vnc console. Ref #1438 +* Support tigervnc in Docker VM. Ref #1438 +* Update minimum VIX version requirements for VMware. Ref #1415. + ## 2.1.11 28/09/2018 * Catch some exceptions. diff --git a/gns3server/crash_report.py b/gns3server/crash_report.py index 26071d86..ff631963 100644 --- a/gns3server/crash_report.py +++ b/gns3server/crash_report.py @@ -57,7 +57,7 @@ class CrashReport: Report crash to a third party service """ - DSN = "https://8a4a7325dfcf4661a0b04d92b0a7d32e:14f83f7a65e54df88e5f06abad85b152@sentry.io/38482" + DSN = "https://edb72a54588b4d04afc2de56b8cb5b24:7830885c8a2a4d0cb311de10ddfe3d27@sentry.io/38482" if hasattr(sys, "frozen"): cacert = get_resource("cacert.pem") if cacert is not None and os.path.isfile(cacert): diff --git a/gns3server/version.py b/gns3server/version.py index 64ba05ae..905e99c3 100644 --- a/gns3server/version.py +++ b/gns3server/version.py @@ -23,8 +23,8 @@ # or negative for a release candidate or beta (after the base version # number has been incremented) -__version__ = "2.1.12dev1" -__version_info__ = (2, 1, 12, 99) +__version__ = "2.1.12" +__version_info__ = (2, 1, 12, 0) # If it's a git checkout try to add the commit if "dev" in __version__: From 7fb192699b40c33afcf33371f403b9f341aa719b Mon Sep 17 00:00:00 2001 From: grossmj Date: Wed, 23 Jan 2019 15:42:10 +0800 Subject: [PATCH 04/17] Development on 2.1.13dev1 --- gns3server/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gns3server/version.py b/gns3server/version.py index 905e99c3..5c7e2b08 100644 --- a/gns3server/version.py +++ b/gns3server/version.py @@ -23,8 +23,8 @@ # or negative for a release candidate or beta (after the base version # number has been incremented) -__version__ = "2.1.12" -__version_info__ = (2, 1, 12, 0) +__version__ = "2.1.13dev1" +__version_info__ = (2, 1, 13, 9) # If it's a git checkout try to add the commit if "dev" in __version__: From 0b07299472e1dc3e3adde386cd0b9ff33067f72b Mon Sep 17 00:00:00 2001 From: grossmj Date: Sun, 17 Feb 2019 19:03:36 +0800 Subject: [PATCH 05/17] Fixes double display output in GRUB in QEMU v3.1. Fixes #1516. --- gns3server/compute/qemu/__init__.py | 2 +- gns3server/compute/qemu/qemu_vm.py | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/gns3server/compute/qemu/__init__.py b/gns3server/compute/qemu/__init__.py index 4608fdd4..c11ceb81 100644 --- a/gns3server/compute/qemu/__init__.py +++ b/gns3server/compute/qemu/__init__.py @@ -186,7 +186,7 @@ class Qemu(BaseManager): return "" else: try: - output = yield from subprocess_check_output(qemu_path, "-version") + output = yield from subprocess_check_output(qemu_path, "-version", "-nographic") match = re.search("version\s+([0-9a-z\-\.]+)", output) if match: version = match.group(1) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 16c9ebdf..4ee17f8a 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1601,18 +1601,19 @@ class QemuVM(BaseNode): return network_options - def _graphic(self): + @asyncio.coroutine + def _disable_graphics(self): """ - Adds the correct graphic options depending of the OS + Disable graphics depending of the QEMU version """ - if sys.platform.startswith("win"): - return [] - if len(os.environ.get("DISPLAY", "")) > 0: + if any(opt in self._options for opt in ["-display", "-nographic", "-curses", "-sdl" "-spice", "-vnc"]): return [] - if "-nographic" not in self._options: + version = yield from self.manager.get_qemu_version(self.qemu_path) + if version and parse_version(version) >= parse_version("3.0"): + return ["-display", "none"] + else: return ["-nographic"] - return [] def _run_with_kvm(self, qemu_path, options): """ @@ -1678,7 +1679,8 @@ class QemuVM(BaseNode): raise QemuError("Console type {} is unknown".format(self._console_type)) command.extend(self._monitor_options()) command.extend((yield from self._network_options())) - command.extend(self._graphic()) + if self._console_type == "telnet": + command.extend((yield from self._disable_graphics())) if additional_options: try: command.extend(shlex.split(additional_options)) From d9a9abf84505b269c93e55e8ff50aae527623e62 Mon Sep 17 00:00:00 2001 From: grossmj Date: Sun, 17 Feb 2019 19:21:21 +0800 Subject: [PATCH 06/17] Add explicit error when trying to pull a Docker image from Docker Hub without Internet access. Fixes #1506. --- gns3server/compute/docker/__init__.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gns3server/compute/docker/__init__.py b/gns3server/compute/docker/__init__.py index f7ee075e..87152338 100644 --- a/gns3server/compute/docker/__init__.py +++ b/gns3server/compute/docker/__init__.py @@ -200,7 +200,10 @@ class Docker(BaseManager): if progress_callback: progress_callback("Pulling '{}' from docker hub".format(image)) - response = yield from self.http_query("POST", "images/create", params={"fromImage": image}, timeout=None) + try: + response = yield from self.http_query("POST", "images/create", params={"fromImage": image}, timeout=None) + except DockerError as e: + raise DockerError("Could not pull the '{}' image from Docker Hub, please check your Internet connection (original error: {})".format(image, e)) # The pull api will stream status via an HTTP JSON stream content = "" while True: From 174624121d07f0c0a24992037bf190c04e36375a Mon Sep 17 00:00:00 2001 From: grossmj Date: Sun, 17 Feb 2019 19:53:46 +0800 Subject: [PATCH 07/17] Fix Qemu VM tests. Ref #1516 --- tests/compute/qemu/test_qemu_vm.py | 60 ++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/tests/compute/qemu/test_qemu_vm.py b/tests/compute/qemu/test_qemu_vm.py index 07dbed25..dd61f3b2 100644 --- a/tests/compute/qemu/test_qemu_vm.py +++ b/tests/compute/qemu/test_qemu_vm.py @@ -116,6 +116,7 @@ def test_is_running(vm, running_subprocess_mock): def test_start(loop, vm, running_subprocess_mock): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") with asyncio_patch("gns3server.compute.qemu.QemuVM.start_wrap_console"): with asyncio_patch("asyncio.create_subprocess_exec", return_value=running_subprocess_mock) as mock: loop.run_until_complete(asyncio.ensure_future(vm.start())) @@ -130,6 +131,7 @@ def test_stop(loop, vm, running_subprocess_mock): future = asyncio.Future() future.set_result(True) process.wait.return_value = future + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") with asyncio_patch("gns3server.compute.qemu.QemuVM.start_wrap_console"): with asyncio_patch("asyncio.create_subprocess_exec", return_value=process): @@ -213,6 +215,7 @@ def test_port_remove_nio_binding(vm, loop): def test_close(vm, port_manager, loop): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") with asyncio_patch("gns3server.compute.qemu.QemuVM.start_wrap_console"): with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()): loop.run_until_complete(asyncio.ensure_future(vm.start())) @@ -338,6 +341,7 @@ def test_disk_options(vm, tmpdir, loop, fake_qemu_img_binary): def test_cdrom_option(vm, tmpdir, loop, fake_qemu_img_binary): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") vm._cdrom_image = str(tmpdir / "test.iso") open(vm._cdrom_image, "w+").close() @@ -348,6 +352,7 @@ def test_cdrom_option(vm, tmpdir, loop, fake_qemu_img_binary): def test_bios_option(vm, tmpdir, loop, fake_qemu_img_binary): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") vm._bios_image = str(tmpdir / "test.img") open(vm._bios_image, "w+").close() @@ -454,6 +459,7 @@ def test_control_vm_expect_text(vm, loop, running_subprocess_mock): def test_build_command(vm, loop, fake_qemu_binary, port_manager): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") os.environ["DISPLAY"] = "0:0" with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: cmd = loop.run_until_complete(asyncio.ensure_future(vm._build_command())) @@ -477,7 +483,9 @@ def test_build_command(vm, loop, fake_qemu_binary, port_manager): "-device", "e1000,mac={},netdev=gns3-0".format(vm._mac_address), "-netdev", - "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport) + "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport), + "-display", + "none" ] @@ -486,6 +494,7 @@ def test_build_command_manual_uuid(vm, loop, fake_qemu_binary, port_manager): If user has set a uuid we keep it """ + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") vm.options = "-uuid e1c307a4-896f-11e6-81a5-3c07547807cc" os.environ["DISPLAY"] = "0:0" with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: @@ -524,7 +533,8 @@ def test_build_command_kvm(linux_platform, vm, loop, fake_qemu_binary, port_mana "-device", "e1000,mac={},netdev=gns3-0".format(vm._mac_address), "-netdev", - "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport) + "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport), + "-nographic" ] @@ -560,13 +570,52 @@ def test_build_command_kvm_2_4(linux_platform, vm, loop, fake_qemu_binary, port_ "-device", "e1000,mac={},netdev=gns3-0".format(vm._mac_address), "-netdev", - "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport) + "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport), + "-nographic" ] +def test_build_command_kvm_3_1(linux_platform, vm, loop, fake_qemu_binary, port_manager): + """ + Qemu 2.4 introduce an issue with KVM + """ + vm._run_with_kvm = MagicMock(return_value=True) + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="3.1.0") + os.environ["DISPLAY"] = "0:0" + with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: + cmd = loop.run_until_complete(asyncio.ensure_future(vm._build_command())) + nio = vm._local_udp_tunnels[0][0] + assert cmd == [ + fake_qemu_binary, + "-name", + "test", + "-m", + "256M", + "-smp", + "cpus=1", + "-enable-kvm", + "-machine", + "smm=off", + "-boot", + "order=c", + "-uuid", + vm.id, + "-serial", + "telnet:127.0.0.1:{},server,nowait".format(vm._internal_console_port), + "-net", + "none", + "-device", + "e1000,mac={},netdev=gns3-0".format(vm._mac_address), + "-netdev", + "socket,id=gns3-0,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio.rport, nio.lport), + "-display", + "none" + ] + @pytest.mark.skipif(sys.platform.startswith("win"), reason="Not supported on Windows") def test_build_command_without_display(vm, loop, fake_qemu_binary): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="2.5.0") os.environ["DISPLAY"] = "" with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: cmd = loop.run_until_complete(asyncio.ensure_future(vm._build_command())) @@ -575,6 +624,7 @@ def test_build_command_without_display(vm, loop, fake_qemu_binary): def test_build_command_two_adapters(vm, loop, fake_qemu_binary, port_manager): + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="2.5.0") os.environ["DISPLAY"] = "0:0" vm.adapters = 2 with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process: @@ -604,7 +654,8 @@ def test_build_command_two_adapters(vm, loop, fake_qemu_binary, port_manager): "-device", "e1000,mac={},netdev=gns3-1".format(int_to_macaddress(macaddress_to_int(vm._mac_address) + 1)), "-netdev", - "socket,id=gns3-1,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio2.rport, nio2.lport) + "socket,id=gns3-1,udp=127.0.0.1:{},localaddr=127.0.0.1:{}".format(nio2.rport, nio2.lport), + "-nographic" ] @@ -613,6 +664,7 @@ def test_build_command_two_adapters_mac_address(vm, loop, fake_qemu_binary, port Should support multiple base vmac address """ + vm.manager.get_qemu_version = AsyncioMagicMock(return_value="2.5.0") vm.adapters = 2 vm.mac_address = "00:00:ab:0e:0f:09" mac_0 = vm._mac_address From 84ee3263ba01019d2e6112395c5804c299296463 Mon Sep 17 00:00:00 2001 From: grossmj Date: Sun, 17 Feb 2019 23:07:33 +0800 Subject: [PATCH 08/17] Count logical CPUs to detect if the number of vCPUs is too high when configuring the GNS3 VM. Fixes #2688. --- gns3server/controller/gns3vm/vmware_gns3_vm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gns3server/controller/gns3vm/vmware_gns3_vm.py b/gns3server/controller/gns3vm/vmware_gns3_vm.py index c14f0732..593e7037 100644 --- a/gns3server/controller/gns3vm/vmware_gns3_vm.py +++ b/gns3server/controller/gns3vm/vmware_gns3_vm.py @@ -72,7 +72,7 @@ class VMwareGNS3VM(BaseGNS3VM): if ram % 4 != 0: raise GNS3VMError("Allocated memory {} for the GNS3 VM must be a multiple of 4".format(ram)) - available_vcpus = psutil.cpu_count(logical=False) + available_vcpus = psutil.cpu_count() if vcpus > available_vcpus: raise GNS3VMError("You have allocated too many vCPUs for the GNS3 VM! (max available is {} vCPUs)".format(available_vcpus)) From 4ecd3b2015fed9bff2bcdc10be9d09ab19f93b29 Mon Sep 17 00:00:00 2001 From: grossmj Date: Sun, 17 Feb 2019 23:16:48 +0800 Subject: [PATCH 09/17] Configure coresPerSocket value in VMX file for the GNS3 VM. Fixes https://github.com/GNS3/gns3-gui/issues/2688 --- gns3server/controller/gns3vm/vmware_gns3_vm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gns3server/controller/gns3vm/vmware_gns3_vm.py b/gns3server/controller/gns3vm/vmware_gns3_vm.py index 593e7037..cd1915b9 100644 --- a/gns3server/controller/gns3vm/vmware_gns3_vm.py +++ b/gns3server/controller/gns3vm/vmware_gns3_vm.py @@ -76,9 +76,11 @@ class VMwareGNS3VM(BaseGNS3VM): if vcpus > available_vcpus: raise GNS3VMError("You have allocated too many vCPUs for the GNS3 VM! (max available is {} vCPUs)".format(available_vcpus)) + cores_per_sockets = int(available_vcpus / psutil.cpu_count(logical=False)) try: pairs = VMware.parse_vmware_file(self._vmx_path) pairs["numvcpus"] = str(vcpus) + pairs["cpuid.coresPerSocket"] = str(cores_per_sockets) pairs["memsize"] = str(ram) VMware.write_vmx_file(self._vmx_path, pairs) log.info("GNS3 VM vCPU count set to {} and RAM amount set to {}".format(vcpus, ram)) From 589c9754e8707cd729aa8106c17f24ad705aa53c Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 19 Feb 2019 00:09:59 +0800 Subject: [PATCH 10/17] Fix symlink not being created for duplicated IOU devices. Fixes https://github.com/GNS3/gns3-gui/issues/2699 --- gns3server/compute/base_manager.py | 2 +- gns3server/compute/iou/iou_vm.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gns3server/compute/base_manager.py b/gns3server/compute/base_manager.py index 602b0017..ca2fa016 100644 --- a/gns3server/compute/base_manager.py +++ b/gns3server/compute/base_manager.py @@ -276,7 +276,7 @@ class BaseManager: destination_dir = destination_node.working_dir try: shutil.rmtree(destination_dir) - shutil.copytree(source_node.working_dir, destination_dir) + shutil.copytree(source_node.working_dir, destination_dir, symlinks=True, ignore_dangling_symlinks=True) except OSError as e: raise aiohttp.web.HTTPConflict(text="Can't duplicate node data: {}".format(e)) diff --git a/gns3server/compute/iou/iou_vm.py b/gns3server/compute/iou/iou_vm.py index 6b0e7c30..53cfe25d 100644 --- a/gns3server/compute/iou/iou_vm.py +++ b/gns3server/compute/iou/iou_vm.py @@ -522,8 +522,9 @@ class IOUVM(BaseNode): # on newer images, see https://github.com/GNS3/gns3-server/issues/1484 try: symlink = os.path.join(self.working_dir, os.path.basename(self.path)) - if not os.path.islink(symlink): - os.symlink(self.path, symlink) + if os.path.islink(symlink): + os.unlink(symlink) + os.symlink(self.path, symlink) except OSError as e: raise IOUError("Could not create symbolic link: {}".format(e)) From ae3515434c4ae10d8b64b91d3e1ab3335cd76f84 Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 19 Feb 2019 12:43:44 +0700 Subject: [PATCH 11/17] Do not export/import symlinks for projects. Fixes #2699 --- gns3server/controller/export_project.py | 7 ++++++- gns3server/controller/import_project.py | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/gns3server/controller/export_project.py b/gns3server/controller/export_project.py index b8cd5a0c..4e376e50 100644 --- a/gns3server/controller/export_project.py +++ b/gns3server/controller/export_project.py @@ -64,7 +64,7 @@ def export_project(project, temporary_dir, include_images=False, keep_compute_id yield from _patch_project_file(project, os.path.join(project._path, file), zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir) # Export the local files - for root, dirs, files in os.walk(project._path, topdown=True): + for root, dirs, files in os.walk(project._path, topdown=True, followlinks=False): files = [f for f in files if _is_exportable(os.path.join(root, f))] for file in files: path = os.path.join(root, file) @@ -125,6 +125,7 @@ def _patch_mtime(path): new_mtime = file_date.replace(year=1980).timestamp() os.utime(path, (st.st_atime, new_mtime)) + def _is_exportable(path): """ :returns: True if file should not be included in the final archive @@ -134,6 +135,10 @@ def _is_exportable(path): if path.endswith("snapshots"): return False + # do not export symlinks + if os.path.islink(path): + return False + # do not export directories of snapshots if "{sep}snapshots{sep}".format(sep=os.path.sep) in path: return False diff --git a/gns3server/controller/import_project.py b/gns3server/controller/import_project.py index a5a89661..83603c17 100644 --- a/gns3server/controller/import_project.py +++ b/gns3server/controller/import_project.py @@ -184,9 +184,11 @@ def _move_files_to_compute(compute, project_id, directory, files_path): location = os.path.join(directory, files_path) if os.path.exists(location): - for (dirpath, dirnames, filenames) in os.walk(location): + for (dirpath, dirnames, filenames) in os.walk(location, followlinks=False): for filename in filenames: path = os.path.join(dirpath, filename) + if os.path.islink(path): + continue dst = os.path.relpath(path, directory) yield from _upload_file(compute, project_id, path, dst) yield from wait_run_in_executor(shutil.rmtree, os.path.join(directory, files_path)) @@ -213,9 +215,11 @@ def _import_images(controller, path): image_dir = controller.images_path() root = os.path.join(path, "images") - for (dirpath, dirnames, filenames) in os.walk(root): + for (dirpath, dirnames, filenames) in os.walk(root, followlinks=False): for filename in filenames: path = os.path.join(dirpath, filename) + if os.path.islink(path): + continue dst = os.path.join(image_dir, os.path.relpath(path, root)) os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.move(path, dst) From aea4ae808f18a1031f0b438a7918658882893d6b Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 19 Feb 2019 17:34:10 +0700 Subject: [PATCH 12/17] Detect invalid environment variable and send a warning when creating a Docker node. Ref #2683 --- gns3server/compute/docker/docker_vm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 1facaff7..b5476020 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -347,6 +347,9 @@ class DockerVM(BaseNode): if self._environment: for e in self._environment.strip().split("\n"): e = e.strip() + if e.split("=")[0] == "": + self.project.emit("log.warning", {"message": "{} has invalid environment variable: {}".format(self.name, e)}) + continue if not e.startswith("GNS3_"): formatted = self._format_env(variables, e) params["Env"].append(formatted) From 081ba31b50f4d0088937611381e767a0fcf47541 Mon Sep 17 00:00:00 2001 From: grossmj Date: Wed, 20 Feb 2019 10:47:33 +0700 Subject: [PATCH 13/17] Fix API call to create a node from an appliance doesn't return the new node data. Fixes #1527 --- gns3server/handlers/api/controller/appliance_handler.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gns3server/handlers/api/controller/appliance_handler.py b/gns3server/handlers/api/controller/appliance_handler.py index b9dbc4cf..ef0a8f40 100644 --- a/gns3server/handlers/api/controller/appliance_handler.py +++ b/gns3server/handlers/api/controller/appliance_handler.py @@ -67,8 +67,9 @@ class ApplianceHandler: controller = Controller.instance() project = controller.get_project(request.match_info["project_id"]) - yield from project.add_node_from_appliance(request.match_info["appliance_id"], - x=request.json["x"], - y=request.json["y"], - compute_id=request.json.get("compute_id")) + node = yield from project.add_node_from_appliance(request.match_info["appliance_id"], + x=request.json["x"], + y=request.json["y"], + compute_id=request.json.get("compute_id")) response.set_status(201) + response.json(node) From 657698a961589cfe1d5f450ff73489b65897fcef Mon Sep 17 00:00:00 2001 From: grossmj Date: Wed, 20 Feb 2019 11:21:29 +0700 Subject: [PATCH 14/17] Fix create a node from an appliance test. --- tests/handlers/api/controller/test_appliance.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/handlers/api/controller/test_appliance.py b/tests/handlers/api/controller/test_appliance.py index 0758b4a5..08666f3a 100644 --- a/tests/handlers/api/controller/test_appliance.py +++ b/tests/handlers/api/controller/test_appliance.py @@ -75,14 +75,13 @@ def test_create_node_from_appliance(http_controller, controller, project, comput "name": "test", "symbol": "guest.svg", "default_name_format": "{name}-{0}", - "server": "example.com" + "compute_id": "example.com" })} - with asyncio_patch("gns3server.controller.project.Project.add_node_from_appliance") as mock: + with asyncio_patch("gns3server.controller.project.Project.add_node_from_appliance", return_value={"name": "test", "node_type": "qemu", "compute_id": "example.com"}) as mock: response = http_controller.post("/projects/{}/appliances/{}".format(project.id, id), { "x": 42, "y": 12 }) mock.assert_called_with(id, x=42, y=12, compute_id=None) - print(response.body) assert response.route == "/projects/{project_id}/appliances/{appliance_id}" assert response.status == 201 From 1ef1872f8e9a09bfc7700916bb19639dfc2ccc96 Mon Sep 17 00:00:00 2001 From: grossmj Date: Wed, 20 Feb 2019 16:38:43 +0700 Subject: [PATCH 15/17] Reset MAC addresses when duplicating a project. Fixes #1522 --- gns3server/controller/export_project.py | 11 ++++++++--- gns3server/controller/project.py | 2 +- gns3server/schemas/dynamips_vm.py | 6 +++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/gns3server/controller/export_project.py b/gns3server/controller/export_project.py index 4e376e50..96212ce2 100644 --- a/gns3server/controller/export_project.py +++ b/gns3server/controller/export_project.py @@ -31,7 +31,7 @@ log = logging.getLogger(__name__) @asyncio.coroutine -def export_project(project, temporary_dir, include_images=False, keep_compute_id=False, allow_all_nodes=False): +def export_project(project, temporary_dir, include_images=False, keep_compute_id=False, allow_all_nodes=False, reset_mac_addresses=False): """ Export a project to a zip file. @@ -42,6 +42,7 @@ def export_project(project, temporary_dir, include_images=False, keep_compute_id :param include images: save OS images to the zip file :param keep_compute_id: If false replace all compute id by local (standard behavior for .gns3project to make it portable) :param allow_all_nodes: Allow all nodes type to be include in the zip even if not portable + :param reset_mac_addresses: Reset MAC addresses for every nodes. :returns: ZipStream object """ @@ -61,7 +62,7 @@ def export_project(project, temporary_dir, include_images=False, keep_compute_id # First we process the .gns3 in order to be sure we don't have an error for file in os.listdir(project._path): if file.endswith(".gns3"): - yield from _patch_project_file(project, os.path.join(project._path, file), zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir) + yield from _patch_project_file(project, os.path.join(project._path, file), zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir, reset_mac_addresses) # Export the local files for root, dirs, files in os.walk(project._path, topdown=True, followlinks=False): @@ -160,7 +161,7 @@ def _is_exportable(path): @asyncio.coroutine -def _patch_project_file(project, path, zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir): +def _patch_project_file(project, path, zstream, include_images, keep_compute_id, allow_all_nodes, temporary_dir, reset_mac_addresses): """ Patch a project file (.gns3) to export a project. The .gns3 file is renamed to project.gns3 @@ -193,6 +194,10 @@ def _patch_project_file(project, path, zstream, include_images, keep_compute_id, if "properties" in node and node["node_type"] != "docker": for prop, value in node["properties"].items(): + # reset the MAC address + if reset_mac_addresses and prop in ("mac_addr", "mac_address"): + node["properties"][prop] = None + if node["node_type"] == "iou": if not prop == "path": continue diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 8c711e7d..de2b516f 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -929,7 +929,7 @@ class Project: self.dump() try: with tempfile.TemporaryDirectory() as tmpdir: - zipstream = yield from export_project(self, tmpdir, keep_compute_id=True, allow_all_nodes=True) + zipstream = yield from export_project(self, tmpdir, keep_compute_id=True, allow_all_nodes=True, reset_mac_addresses=True) project_path = os.path.join(tmpdir, "project.gns3p") yield from wait_run_in_executor(self._create_duplicate_project_file, project_path, zipstream) with open(project_path, "rb") as f: diff --git a/gns3server/schemas/dynamips_vm.py b/gns3server/schemas/dynamips_vm.py index 43b7da4f..98441f1d 100644 --- a/gns3server/schemas/dynamips_vm.py +++ b/gns3server/schemas/dynamips_vm.py @@ -137,7 +137,7 @@ VM_CREATE_SCHEMA = { }, "mac_addr": { "description": "Base MAC address", - "type": "string", + "type": ["null", "string"], "minLength": 1, "pattern": "^([0-9a-fA-F]{4}\\.){2}[0-9a-fA-F]{4}$" }, @@ -355,7 +355,7 @@ VM_UPDATE_SCHEMA = { }, "mac_addr": { "description": "Base MAC address", - "type": "string", + "type": ["null", "string"], "minLength": 1, "pattern": "^([0-9a-fA-F]{4}\\.){2}[0-9a-fA-F]{4}$" }, @@ -595,7 +595,7 @@ VM_OBJECT_SCHEMA = { }, "mac_addr": { "description": "Base MAC address", - "type": "string", + "type": ["null", "string"] #"minLength": 1, #"pattern": "^([0-9a-fA-F]{4}\\.){2}[0-9a-fA-F]{4}$" }, From a13d063aa12736d87cd4e217bb6755744775e755 Mon Sep 17 00:00:00 2001 From: grossmj Date: Thu, 21 Feb 2019 23:58:54 +0700 Subject: [PATCH 16/17] Fix topology images (Pictures) disappearing from projects. Fixes #1514. --- gns3server/controller/drawing.py | 2 +- gns3server/controller/project.py | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/gns3server/controller/drawing.py b/gns3server/controller/drawing.py index 47c09d0e..5db6e5c3 100644 --- a/gns3server/controller/drawing.py +++ b/gns3server/controller/drawing.py @@ -55,7 +55,7 @@ class Drawing: return self._id @property - def ressource_filename(self): + def resource_filename(self): """ If the svg content has been dump to an external file return is name otherwise None """ diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index de2b516f..ebcb89fd 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -146,7 +146,6 @@ class Project: } ) - def reset(self): """ Called when open/close a project. Cleanup internal stuff @@ -704,24 +703,26 @@ class Project: # We don't care if a compute is down at this step except (ComputeError, aiohttp.web.HTTPError, aiohttp.ClientResponseError, TimeoutError): pass - self._cleanPictures() + self._clean_pictures() self._status = "closed" if not ignore_notification: self.controller.notification.emit("project.closed", self.__json__()) - def _cleanPictures(self): + def _clean_pictures(self): """ - Delete unused images + Delete unused pictures. """ - # Project have been deleted - if not os.path.exists(self.path): + # Project have been deleted or is loading or is not opened + if not os.path.exists(self.path) or self._loading or self._status != "opened": return try: pictures = set(os.listdir(self.pictures_directory)) for drawing in self._drawings.values(): try: - pictures.remove(drawing.ressource_filename) + resource_filename = drawing.resource_filename + if resource_filename: + pictures.remove(resource_filename) except KeyError: pass @@ -733,11 +734,12 @@ class Project: except KeyError: pass - - for pict in pictures: - os.remove(os.path.join(self.pictures_directory, pict)) + for pic_filename in pictures: + path = os.path.join(self.pictures_directory, pic_filename) + log.info("Deleting unused picture '{}'".format(path)) + os.remove(path) except OSError as e: - log.warning(str(e)) + log.warning("Could not delete unused pictures: {}".format(e)) @asyncio.coroutine def delete(self): From 62c51edbaef9e4a58eeb98f46019cedcbdb19112 Mon Sep 17 00:00:00 2001 From: grossmj Date: Fri, 22 Feb 2019 16:05:31 +0700 Subject: [PATCH 17/17] Less aggressive connections to uBridge. Ref #1289 --- gns3server/ubridge/ubridge_hypervisor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gns3server/ubridge/ubridge_hypervisor.py b/gns3server/ubridge/ubridge_hypervisor.py index 092189ee..7f050723 100644 --- a/gns3server/ubridge/ubridge_hypervisor.py +++ b/gns3server/ubridge/ubridge_hypervisor.py @@ -69,7 +69,7 @@ class UBridgeHypervisor: connection_success = False last_exception = None while time.time() - begin < timeout: - yield from asyncio.sleep(0.01) + yield from asyncio.sleep(0.1) try: self._reader, self._writer = yield from asyncio.open_connection(host, self._port) except OSError as e: @@ -84,6 +84,7 @@ class UBridgeHypervisor: log.info("Connected to uBridge hypervisor after {:.4f} seconds".format(time.time() - begin)) try: + yield from asyncio.sleep(0.1) version = yield from self.send("hypervisor version") self._version = version[0].split("-", 1)[0] except IndexError: @@ -237,7 +238,7 @@ class UBridgeHypervisor: .format(host=self._host, port=self._port, command=command, run=self.is_running())) else: retries += 1 - yield from asyncio.sleep(0.1) + yield from asyncio.sleep(0.5) continue retries = 0 buf += chunk.decode("utf-8")