Merge pull request #2133 from GNS3/use-importlib-resources

Migrate to importlib_resources

gns3server/controller/__init__.py

@@ -22,6 +22,7 @@ import uuid
 import socket
 import shutil
 import aiohttp
+import importlib_resources
 
 from ..config import Config
 from .project import Project
@@ -35,7 +36,6 @@ from .symbols import Symbols
 from ..version import __version__
 from .topology import load_topology
 from .gns3vm import GNS3VM
-from ..utils.get_resource import get_resource
 from .gns3vm.gns3_vm_error import GNS3VMError
 
 import logging
@@ -65,7 +65,7 @@ class Controller:
     async def start(self):
 
         log.info("Controller is starting")
-        self.load_base_files()
+        self._load_base_files()
         server_config = Config.instance().get_section_config("Server")
         Config.instance().listen_for_config_changes(self._update_config)
         host = server_config.get("host", "localhost")
@@ -242,6 +242,7 @@ class Controller:
         if "iou_license" in controller_settings:
             self._iou_license_settings = controller_settings["iou_license"]
 
+        self._appliance_manager.install_builtin_appliances()
         self._appliance_manager.appliances_etag = controller_settings.get("appliances_etag")
         self._appliance_manager.load_appliances()
         self._template_manager.load_templates(controller_settings.get("templates"))
@@ -269,20 +270,21 @@ class Controller:
         except OSError as e:
             log.error(str(e))
 
-    def load_base_files(self):
+    def _load_base_files(self):
         """
         At startup we copy base file to the user location to allow
         them to customize it
         """
 
         dst_path = self.configs_path()
-        src_path = get_resource('configs')
         try:
-            for file in os.listdir(src_path):
-                if not os.path.exists(os.path.join(dst_path, file)):
-                    shutil.copy(os.path.join(src_path, file), os.path.join(dst_path, file))
-        except OSError:
-            pass
+            for entry in importlib_resources.files('gns3server.configs').iterdir():
+                full_path = os.path.join(dst_path, entry.name)
+                if entry.is_file() and not os.path.exists(full_path):
+                    log.debug(f"Installing base config file {entry.name} to {full_path}")
+                    shutil.copy(str(entry), os.path.join(dst_path, entry.name))
+        except OSError as e:
+            log.error(f"Could not install base config files to {dst_path}: {e}")
 
     def images_path(self):
         """

gns3server/controller/appliance_manager.py

@@ -16,11 +16,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import shutil
 import json
 import uuid
 import asyncio
 import aiohttp
+import importlib_resources
+import shutil
 
 from .appliance import Appliance
 from ..config import Config
@@ -65,9 +66,9 @@ class ApplianceManager:
 
         return self._appliances
 
-    def appliances_path(self):
+    def _custom_appliances_path(self):
         """
-        Get the image storage directory
+        Get the custom appliance storage directory
         """
 
         server_config = Config.instance().get_section_config("Server")
@@ -75,13 +76,38 @@ class ApplianceManager:
         os.makedirs(appliances_path, exist_ok=True)
         return appliances_path
 
+    def _builtin_appliances_path(self):
+        """
+        Get the built-in appliance storage directory
+        """
+
+        config = Config.instance()
+        appliances_dir = os.path.join(config.config_dir, "appliances")
+        os.makedirs(appliances_dir, exist_ok=True)
+        return appliances_dir
+
+    def install_builtin_appliances(self):
+        """
+        At startup we copy the built-in appliances files.
+        """
+
+        dst_path = self._builtin_appliances_path()
+        try:
+            for entry in importlib_resources.files('gns3server.appliances').iterdir():
+                full_path = os.path.join(dst_path, entry.name)
+                if entry.is_file() and not os.path.exists(full_path):
+                    log.debug(f"Installing built-in appliance file {entry.name} to {full_path}")
+                    shutil.copy(str(entry), os.path.join(dst_path, entry.name))
+        except OSError as e:
+            log.error(f"Could not install built-in appliance files to {dst_path}: {e}")
+
     def load_appliances(self, symbol_theme="Classic"):
         """
         Loads appliance files from disk.
         """
 
         self._appliances = {}
-        for directory, builtin in ((get_resource('appliances'), True,), (self.appliances_path(), False,)):
+        for directory, builtin in ((self._builtin_appliances_path(), True,), (self._custom_appliances_path(), False,)):
             if directory and os.path.isdir(directory):
                 for file in os.listdir(directory):
                     if not file.endswith('.gns3a') and not file.endswith('.gns3appliance'):
@@ -181,7 +207,7 @@ class ApplianceManager:
                         from . import Controller
                         Controller.instance().save()
                     json_data = await response.json()
-                appliances_dir = get_resource('appliances')
+                appliances_dir = self._builtin_appliances_path()
                 downloaded_appliance_files = []
                 for appliance in json_data:
                     if appliance["type"] == "file":

gns3server/utils/get_resource.py

@@ -15,33 +15,18 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import tempfile
-import pkg_resources
 import atexit
 import logging
 import os
 import sys
+import importlib_resources
+
+from contextlib import ExitStack
+resource_manager = ExitStack()
+atexit.register(resource_manager.close)
 
 log = logging.getLogger(__name__)
 
-try:
-    egg_cache_dir = tempfile.mkdtemp()
-    pkg_resources.set_extraction_path(egg_cache_dir)
-except ValueError:
-    # If the path is already set the module throw an error
-    pass
-
-
-@atexit.register
-def clean_egg_cache():
-    try:
-        import shutil
-        log.debug("Clean egg cache %s", egg_cache_dir)
-        shutil.rmtree(egg_cache_dir)
-    except Exception:
-        # We don't care if we can not cleanup
-        pass
-
 
 def get_resource(resource_name):
     """
@@ -51,7 +36,9 @@ def get_resource(resource_name):
     resource_path = None
     if hasattr(sys, "frozen"):
         resource_path = os.path.normpath(os.path.join(os.path.dirname(sys.executable), resource_name))
-    elif not hasattr(sys, "frozen") and pkg_resources.resource_exists("gns3server", resource_name):
-        resource_path = pkg_resources.resource_filename("gns3server", resource_name)
-        resource_path = os.path.normpath(resource_path)
+    else:
+        ref = importlib_resources.files("gns3server") / resource_name
+        path = resource_manager.enter_context(importlib_resources.as_file(ref))
+        if os.path.exists(path):
+            resource_path = os.path.normpath(path)
     return resource_path

requirements.txt

@@ -11,5 +11,6 @@ psutil>=5.9.2,<5.10
 async-timeout>=4.0.2,<4.1
 distro>=1.7.0
 py-cpuinfo>=8.0.0,<8.1
+importlib-resources>=1.3
 setuptools>=60.8.1; python_version >= '3.7'
 setuptools==59.6.0; python_version < '3.7'  # v59.6.0 is the last version to support Python 3.6

tests/controller/test_controller.py

@@ -387,10 +387,10 @@ async def test_load_base_files(controller, config, tmpdir):
     with open(str(tmpdir / 'iou_l2_base_startup-config.txt'), 'w+') as f:
         f.write('test')
 
-    controller.load_base_files()
+    controller._load_base_files()
     assert os.path.exists(str(tmpdir / 'iou_l3_base_startup-config.txt'))
 
-    # Check is the file has not been overwrite
+    # Check is the file has not been overwritten
     with open(str(tmpdir / 'iou_l2_base_startup-config.txt')) as f:
         assert f.read() == 'test'