From a4b24eacebf1dc2646606496ec898fbe50706502 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 7 Nov 2022 20:12:03 +0800 Subject: [PATCH 1/2] Use importlib_resources instead of pkg_resources and install built-in appliances in config dir. --- gns3server/controller/__init__.py | 20 ++++++------ gns3server/controller/appliance_manager.py | 36 +++++++++++++++++++--- gns3server/utils/get_resource.py | 33 ++++++-------------- requirements.txt | 1 + 4 files changed, 53 insertions(+), 37 deletions(-) diff --git a/gns3server/controller/__init__.py b/gns3server/controller/__init__.py index e000ea06..a3c16fb3 100644 --- a/gns3server/controller/__init__.py +++ b/gns3server/controller/__init__.py @@ -22,6 +22,7 @@ import uuid import socket import shutil import aiohttp +import importlib_resources from ..config import Config from .project import Project @@ -35,7 +36,6 @@ from .symbols import Symbols from ..version import __version__ from .topology import load_topology from .gns3vm import GNS3VM -from ..utils.get_resource import get_resource from .gns3vm.gns3_vm_error import GNS3VMError import logging @@ -65,7 +65,7 @@ class Controller: async def start(self): log.info("Controller is starting") - self.load_base_files() + self._load_base_files() server_config = Config.instance().get_section_config("Server") Config.instance().listen_for_config_changes(self._update_config) host = server_config.get("host", "localhost") @@ -242,6 +242,7 @@ class Controller: if "iou_license" in controller_settings: self._iou_license_settings = controller_settings["iou_license"] + self._appliance_manager.install_builtin_appliances() self._appliance_manager.appliances_etag = controller_settings.get("appliances_etag") self._appliance_manager.load_appliances() self._template_manager.load_templates(controller_settings.get("templates")) @@ -269,20 +270,21 @@ class Controller: except OSError as e: log.error(str(e)) - def load_base_files(self): + def _load_base_files(self): """ At startup we copy base file to the user location to allow them to customize it """ dst_path = self.configs_path() - src_path = get_resource('configs') try: - for file in os.listdir(src_path): - if not os.path.exists(os.path.join(dst_path, file)): - shutil.copy(os.path.join(src_path, file), os.path.join(dst_path, file)) - except OSError: - pass + for entry in importlib_resources.files('gns3server.configs').iterdir(): + full_path = os.path.join(dst_path, entry.name) + if entry.is_file() and not os.path.exists(full_path): + log.debug(f"Installing base config file {entry.name} to {full_path}") + shutil.copy(str(entry), os.path.join(dst_path, entry.name)) + except OSError as e: + log.error(f"Could not install base config files to {dst_path}: {e}") def images_path(self): """ diff --git a/gns3server/controller/appliance_manager.py b/gns3server/controller/appliance_manager.py index cebcad58..b9349152 100644 --- a/gns3server/controller/appliance_manager.py +++ b/gns3server/controller/appliance_manager.py @@ -16,11 +16,12 @@ # along with this program. If not, see . import os -import shutil import json import uuid import asyncio import aiohttp +import importlib_resources +import shutil from .appliance import Appliance from ..config import Config @@ -65,9 +66,9 @@ class ApplianceManager: return self._appliances - def appliances_path(self): + def _custom_appliances_path(self): """ - Get the image storage directory + Get the custom appliance storage directory """ server_config = Config.instance().get_section_config("Server") @@ -75,13 +76,38 @@ class ApplianceManager: os.makedirs(appliances_path, exist_ok=True) return appliances_path + def _builtin_appliances_path(self): + """ + Get the built-in appliance storage directory + """ + + config = Config.instance() + appliances_dir = os.path.join(config.config_dir, "appliances") + os.makedirs(appliances_dir, exist_ok=True) + return appliances_dir + + def install_builtin_appliances(self): + """ + At startup we copy the built-in appliances files. + """ + + dst_path = self._builtin_appliances_path() + try: + for entry in importlib_resources.files('gns3server.appliances').iterdir(): + full_path = os.path.join(dst_path, entry.name) + if entry.is_file() and not os.path.exists(full_path): + log.debug(f"Installing built-in appliance file {entry.name} to {full_path}") + shutil.copy(str(entry), os.path.join(dst_path, entry.name)) + except OSError as e: + log.error(f"Could not install built-in appliance files to {dst_path}: {e}") + def load_appliances(self, symbol_theme="Classic"): """ Loads appliance files from disk. """ self._appliances = {} - for directory, builtin in ((get_resource('appliances'), True,), (self.appliances_path(), False,)): + for directory, builtin in ((self._builtin_appliances_path(), True,), (self._custom_appliances_path(), False,)): if directory and os.path.isdir(directory): for file in os.listdir(directory): if not file.endswith('.gns3a') and not file.endswith('.gns3appliance'): @@ -181,7 +207,7 @@ class ApplianceManager: from . import Controller Controller.instance().save() json_data = await response.json() - appliances_dir = get_resource('appliances') + appliances_dir = self._builtin_appliances_path() downloaded_appliance_files = [] for appliance in json_data: if appliance["type"] == "file": diff --git a/gns3server/utils/get_resource.py b/gns3server/utils/get_resource.py index b988a200..b4b599bd 100644 --- a/gns3server/utils/get_resource.py +++ b/gns3server/utils/get_resource.py @@ -15,33 +15,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import tempfile -import pkg_resources import atexit import logging import os import sys +import importlib_resources + +from contextlib import ExitStack +resource_manager = ExitStack() +atexit.register(resource_manager.close) log = logging.getLogger(__name__) -try: - egg_cache_dir = tempfile.mkdtemp() - pkg_resources.set_extraction_path(egg_cache_dir) -except ValueError: - # If the path is already set the module throw an error - pass - - -@atexit.register -def clean_egg_cache(): - try: - import shutil - log.debug("Clean egg cache %s", egg_cache_dir) - shutil.rmtree(egg_cache_dir) - except Exception: - # We don't care if we can not cleanup - pass - def get_resource(resource_name): """ @@ -51,7 +36,9 @@ def get_resource(resource_name): resource_path = None if hasattr(sys, "frozen"): resource_path = os.path.normpath(os.path.join(os.path.dirname(sys.executable), resource_name)) - elif not hasattr(sys, "frozen") and pkg_resources.resource_exists("gns3server", resource_name): - resource_path = pkg_resources.resource_filename("gns3server", resource_name) - resource_path = os.path.normpath(resource_path) + else: + ref = importlib_resources.files("gns3server") / resource_name + path = resource_manager.enter_context(importlib_resources.as_file(ref)) + if os.path.exists(path): + resource_path = os.path.normpath(path) return resource_path diff --git a/requirements.txt b/requirements.txt index 735c4eac..5bc393a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,5 +11,6 @@ psutil>=5.9.2,<5.10 async-timeout>=4.0.2,<4.1 distro>=1.7.0 py-cpuinfo>=8.0.0,<8.1 +importlib-resources>=1.3 setuptools>=60.8.1; python_version >= '3.7' setuptools==59.6.0; python_version < '3.7' # v59.6.0 is the last version to support Python 3.6 From 2e550d839e658d5d83a47535ee4931cfaa60bee9 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 7 Nov 2022 20:33:02 +0800 Subject: [PATCH 2/2] Fix tests --- tests/controller/test_controller.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/controller/test_controller.py b/tests/controller/test_controller.py index 48b8d98d..48ba98af 100644 --- a/tests/controller/test_controller.py +++ b/tests/controller/test_controller.py @@ -387,10 +387,10 @@ async def test_load_base_files(controller, config, tmpdir): with open(str(tmpdir / 'iou_l2_base_startup-config.txt'), 'w+') as f: f.write('test') - controller.load_base_files() + controller._load_base_files() assert os.path.exists(str(tmpdir / 'iou_l3_base_startup-config.txt')) - # Check is the file has not been overwrite + # Check is the file has not been overwritten with open(str(tmpdir / 'iou_l2_base_startup-config.txt')) as f: assert f.read() == 'test'