From bae3fb84b94c53cc78b3e5ec9508403d87b05597 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 11 Mar 2019 16:55:16 +0700 Subject: [PATCH] Download custom appliance symbols from GitHub Fix symbol cache issue. Ref https://github.com/GNS3/gns3-gui/issues/2671 Fix temporary directory for symbols was not deleted Fix temporary appliance file was not deleted --- gns3server/controller/appliance.py | 4 ++ gns3server/controller/appliance_manager.py | 48 +++++++++++++++++-- gns3server/controller/symbol_themes.py | 24 +++++----- gns3server/controller/symbols.py | 32 ++++++++----- .../api/controller/template_handler.py | 2 + tests/controller/test_node.py | 9 ++-- tests/controller/test_symbols.py | 8 ++-- tests/handlers/api/controller/test_symbol.py | 5 +- 8 files changed, 95 insertions(+), 37 deletions(-) diff --git a/gns3server/controller/appliance.py b/gns3server/controller/appliance.py index ad951f91..a04b8b38 100644 --- a/gns3server/controller/appliance.py +++ b/gns3server/controller/appliance.py @@ -41,6 +41,10 @@ class Appliance: def status(self): return self._data["status"] + @property + def symbol(self): + return self._data.get("symbol") + def __json__(self): """ Appliance data (a hash) diff --git a/gns3server/controller/appliance_manager.py b/gns3server/controller/appliance_manager.py index d2343879..00819d83 100644 --- a/gns3server/controller/appliance_manager.py +++ b/gns3server/controller/appliance_manager.py @@ -91,12 +91,51 @@ class ApplianceManager: with open(path, 'r', encoding='utf-8') as f: appliance = Appliance(appliance_id, json.load(f), builtin=builtin) appliance.__json__() # Check if loaded without error - if appliance.status != 'broken': - self._appliances[appliance.id] = appliance + if appliance.status != 'broken': + self._appliances[appliance.id] = appliance except (ValueError, OSError, KeyError) as e: log.warning("Cannot load appliance file '%s': %s", path, str(e)) continue + async def download_custom_symbols(self): + """ + Download custom appliance symbols from our GitHub registry repository. + """ + + from . import Controller + symbol_dir = Controller.instance().symbols.symbols_path() + self.load_appliances() + for appliance in self._appliances.values(): + symbol = appliance.symbol + if symbol and not symbol.startswith(":/symbols/"): + destination_path = os.path.join(symbol_dir, symbol) + if not os.path.exists(destination_path): + await self._download_symbol(symbol, destination_path) + + # refresh the symbol cache + Controller.instance().symbols.list() + + async def _download_symbol(self, symbol, destination_path): + """ + Download a custom appliance symbol from our GitHub registry repository. + """ + + symbol_url = "https://raw.githubusercontent.com/GNS3/gns3-registry/master/symbols/{}".format(symbol) + async with aiohttp.ClientSession() as session: + async with session.get(symbol_url) as response: + if response.status != 200: + log.warning("Could not retrieve appliance symbol {} from GitHub due to HTTP error code {}".format(symbol, response.status)) + else: + try: + symbol_data = await response.read() + log.info("Saving {} symbol to {}".format(symbol, destination_path)) + with open(destination_path, 'wb') as f: + f.write(symbol_data) + except asyncio.TimeoutError: + log.warning("Timeout while downloading '{}'".format(symbol_url)) + except OSError as e: + log.warning("Could not write appliance symbol '{}': {}".format(destination_path, e)) + @locking async def download_appliances(self): """ @@ -114,7 +153,7 @@ class ApplianceManager: log.info("Appliances are already up-to-date (ETag {})".format(self._appliances_etag)) return elif response.status != 200: - raise aiohttp.web.HTTPConflict(text="Could not retrieve appliances on GitHub due to HTTP error code {}".format(response.status)) + raise aiohttp.web.HTTPConflict(text="Could not retrieve appliances from GitHub due to HTTP error code {}".format(response.status)) etag = response.headers.get("ETag") if etag: self._appliances_etag = etag @@ -144,3 +183,6 @@ class ApplianceManager: raise aiohttp.web.HTTPConflict(text="Could not write appliance file '{}': {}".format(path, e)) except ValueError as e: raise aiohttp.web.HTTPConflict(text="Could not read appliances information from GitHub: {}".format(e)) + + # download the custom symbols + await self.download_custom_symbols() diff --git a/gns3server/controller/symbol_themes.py b/gns3server/controller/symbol_themes.py index b055fff2..876ad8df 100644 --- a/gns3server/controller/symbol_themes.py +++ b/gns3server/controller/symbol_themes.py @@ -31,7 +31,7 @@ CLASSIC_SYMBOL_THEME = {"cloud": ":/symbols/classic/cloud.svg", "vmware_guest": ":/symbols/classic/vmware_guest.svg", "docker_guest": ":/symbols/classic/docker_guest.svg"} -INFINITY_SQUARE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/blue/cloud.svg", +AFFINITY_SQUARE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/blue/cloud.svg", "ethernet_switch": ":/symbols/affinity/square/blue/switch.svg", "ethernet_hub": ":/symbols/affinity/square/blue/hub.svg", "frame_relay_switch.svg": ":/symbols/affinity/square/blue/isdn.svg", @@ -46,7 +46,7 @@ INFINITY_SQUARE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/blue/cl "vmware_guest": ":/symbols/affinity/square/blue/vmware.svg", "docker_guest": ":/symbols/affinity/square/blue/docker.svg"} -INFINITY_SQUARE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/red/cloud.svg", +AFFINITY_SQUARE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/red/cloud.svg", "ethernet_switch": ":/symbols/affinity/square/red/switch.svg", "ethernet_hub": ":/symbols/affinity/square/red/hub.svg", "frame_relay_switch": ":/symbols/affinity/square/red/isdn.svg", @@ -61,7 +61,7 @@ INFINITY_SQUARE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/red/clou "vmware_guest": ":/symbols/affinity/square/red/vmware.svg", "docker_guest": ":/symbols/affinity/square/red/docker.svg"} -INFINITY_SQUARE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/gray/cloud.svg", +AFFINITY_SQUARE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/gray/cloud.svg", "ethernet_switch": ":/symbols/affinity/square/gray/switch.svg", "ethernet_hub": ":/symbols/affinity/square/gray/hub.svg", "frame_relay_switch": ":/symbols/affinity/square/gray/isdn.svg", @@ -76,7 +76,7 @@ INFINITY_SQUARE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/square/gray/cl "vmware_guest": ":/symbols/affinity/square/gray/vmware.svg", "docker_guest": ":/symbols/affinity/square/gray/docker.svg"} -INFINITY_CIRCLE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/blue/cloud.svg", +AFFINITY_CIRCLE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/blue/cloud.svg", "ethernet_switch": ":/symbols/affinity/circle/blue/switch.svg", "ethernet_hub": ":/symbols/affinity/circle/blue/hub.svg", "frame_relay_switch": ":/symbols/affinity/circle/blue/isdn.svg", @@ -91,7 +91,7 @@ INFINITY_CIRCLE_BLUE_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/blue/cl "vmware_guest": ":/symbols/affinity/circle/blue/vmware.svg", "docker_guest": ":/symbols/affinity/circle/blue/docker.svg"} -INFINITY_CIRCLE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/red/cloud.svg", +AFFINITY_CIRCLE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/red/cloud.svg", "ethernet_switch": ":/symbols/affinity/circle/red/switch.svg", "ethernet_hub": ":/symbols/affinity/circle/red/hub.svg", "frame_relay_switch": ":/symbols/affinity/circle/red/isdn.svg", @@ -106,7 +106,7 @@ INFINITY_CIRCLE_RED_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/red/clou "vmware_guest": ":/symbols/affinity/circle/red/vmware.svg", "docker_guest": ":/symbols/affinity/circle/red/docker.svg"} -INFINITY_CIRCLE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/gray/cloud.svg", +AFFINITY_CIRCLE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/gray/cloud.svg", "ethernet_switch": ":/symbols/affinity/circle/gray/switch.svg", "ethernet_hub": ":/symbols/affinity/circle/gray/hub.svg", "frame_relay_switch": ":/symbols/affinity/circle/gray/isdn.svg", @@ -122,9 +122,9 @@ INFINITY_CIRCLE_GRAY_SYMBOL_THEME = {"cloud": ":/symbols/affinity/circle/gray/cl "docker_guest": ":/symbols/affinity/circle/gray/docker.svg"} BUILTIN_SYMBOL_THEMES = {"Classic": CLASSIC_SYMBOL_THEME, - "Infinity-square-blue": INFINITY_SQUARE_BLUE_SYMBOL_THEME, - "Infinity-square-red": INFINITY_SQUARE_RED_SYMBOL_THEME, - "Infinity-square-gray": INFINITY_SQUARE_GRAY_SYMBOL_THEME, - "Infinity-circle-blue": INFINITY_CIRCLE_BLUE_SYMBOL_THEME, - "Infinity-circle-red": INFINITY_CIRCLE_RED_SYMBOL_THEME, - "Infinity-circle-gray": INFINITY_CIRCLE_GRAY_SYMBOL_THEME} + "Affinity-square-blue": AFFINITY_SQUARE_BLUE_SYMBOL_THEME, + "Affinity-square-red": AFFINITY_SQUARE_RED_SYMBOL_THEME, + "Affinity-square-gray": AFFINITY_SQUARE_GRAY_SYMBOL_THEME, + "Affinity-circle-blue": AFFINITY_CIRCLE_BLUE_SYMBOL_THEME, + "Affinity-circle-red": AFFINITY_CIRCLE_RED_SYMBOL_THEME, + "Affinity-circle-gray": AFFINITY_CIRCLE_GRAY_SYMBOL_THEME} diff --git a/gns3server/controller/symbols.py b/gns3server/controller/symbols.py index 6cbdbdce..fc29ce47 100644 --- a/gns3server/controller/symbols.py +++ b/gns3server/controller/symbols.py @@ -67,9 +67,11 @@ class Symbols: if filename.startswith('.'): continue symbol_file = posixpath.normpath(os.path.relpath(os.path.join(root, filename), get_resource("symbols"))).replace('\\', '/') + theme = posixpath.dirname(symbol_file).replace('/', '-').capitalize() symbol_id = ':/symbols/' + symbol_file symbols.append({'symbol_id': symbol_id, - 'filename': symbol_file, + 'filename': filename, + 'theme': theme, 'builtin': True}) self._symbols_path[symbol_id] = os.path.join(root, filename) @@ -81,8 +83,9 @@ class Symbols: continue symbol_file = posixpath.normpath(os.path.relpath(os.path.join(root, filename), directory)).replace('\\', '/') symbols.append({'symbol_id': symbol_file, - 'filename': symbol_file, - 'builtin': False,}) + 'filename': filename, + 'builtin': False, + 'theme': "Custom symbols"}) self._symbols_path[symbol_file] = os.path.join(root, filename) symbols.sort(key=lambda x: x["filename"]) @@ -99,21 +102,26 @@ class Symbols: return directory def get_path(self, symbol_id): - symbol_filename = os.path.splitext(os.path.basename(symbol_id))[0] - theme = self._themes.get(self._current_theme, {}) - if not theme: - log.error("Could not find symbol theme '{}'".format(self._current_theme)) + #symbol_filename = os.path.splitext(os.path.basename(symbol_id))[0] + #theme = self._themes.get(self._current_theme, {}) + #if not theme: + # log.error("Could not find symbol theme '{}'".format(self._current_theme)) try: - return self._symbols_path[theme.get(symbol_filename, symbol_id)] + return self._symbols_path[symbol_id] + #return self._symbols_path[theme.get(symbol_filename, symbol_id)] except KeyError: - # Symbol not found, let's refresh the cache try: self.list() return self._symbols_path[symbol_id] except (OSError, KeyError): - log.warning("Could not retrieve symbol '{}'".format(symbol_id)) - symbols_path = self._symbols_path - return symbols_path.get(":/symbols/classic/{}".format(os.path.basename(symbol_id)), symbols_path[":/symbols/classic/computer.svg"]) + # try to return a symbol with the same name from the classic theme + symbol = self._symbols_path.get(":/symbols/classic/{}".format(os.path.basename(symbol_id))) + if symbol: + return symbol + else: + # return the default computer symbol + log.warning("Could not retrieve symbol '{}'".format(symbol_id)) + return self._symbols_path[":/symbols/classic/computer.svg"] def get_size(self, symbol_id): try: diff --git a/gns3server/handlers/api/controller/template_handler.py b/gns3server/handlers/api/controller/template_handler.py index 6d6005d2..0f88efb3 100644 --- a/gns3server/handlers/api/controller/template_handler.py +++ b/gns3server/handlers/api/controller/template_handler.py @@ -51,6 +51,8 @@ class TemplateHandler: controller = Controller.instance() template = controller.template_manager.add_template(request.json) + # Reset the symbol list + controller.symbols.list() response.set_status(201) response.json(template) diff --git a/tests/controller/test_node.py b/tests/controller/test_node.py index e72b1db4..4c4e5ec4 100644 --- a/tests/controller/test_node.py +++ b/tests/controller/test_node.py @@ -259,16 +259,15 @@ def test_symbol(node, symbols_dir, controller): Change symbol should change the node size """ - controller.symbols.theme = "Classic" - node.symbol = ":/symbols/dslam.svg" - assert node.symbol == ":/symbols/dslam.svg" + node.symbol = ":/symbols/classic/dslam.svg" + assert node.symbol == ":/symbols/classic/dslam.svg" assert node.width == 50 assert node.height == 53 assert node.label["x"] is None assert node.label["y"] == -40 - node.symbol = ":/symbols/cloud.svg" - assert node.symbol == ":/symbols/cloud.svg" + node.symbol = ":/symbols/classic/cloud.svg" + assert node.symbol == ":/symbols/classic/cloud.svg" assert node.width == 159 assert node.height == 71 assert node.label["x"] is None diff --git a/tests/controller/test_symbols.py b/tests/controller/test_symbols.py index d8c0b9a7..17db40f4 100644 --- a/tests/controller/test_symbols.py +++ b/tests/controller/test_symbols.py @@ -31,13 +31,15 @@ def test_list(symbols_dir): symbols = Symbols() assert { 'symbol_id': ':/symbols/classic/firewall.svg', - 'filename': 'classic/firewall.svg', - 'builtin': True + 'filename': 'firewall.svg', + 'builtin': True, + 'theme': 'Classic' } in symbols.list() assert { 'symbol_id': 'linux.svg', 'filename': 'linux.svg', - 'builtin': False + 'builtin': False, + 'theme': 'Custom symbols' } in symbols.list() diff --git a/tests/handlers/api/controller/test_symbol.py b/tests/handlers/api/controller/test_symbol.py index 5b5c6b6f..c03450ce 100644 --- a/tests/handlers/api/controller/test_symbol.py +++ b/tests/handlers/api/controller/test_symbol.py @@ -27,8 +27,9 @@ def test_symbols(http_controller): assert response.status == 200 assert { 'symbol_id': ':/symbols/classic/firewall.svg', - 'filename': 'classic/firewall.svg', - 'builtin': True + 'filename': 'firewall.svg', + 'builtin': True, + 'theme': 'Classic' } in response.json