From 3106c8a6a2c9ac61f255b1c74d9b44df5cd0afc0 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 18 Apr 2022 17:13:52 +0700 Subject: [PATCH] Do not cache to md5sum file in some situations --- gns3server/compute/qemu/qemu_vm.py | 8 ++++---- gns3server/controller/appliance_manager.py | 8 +++++++- gns3server/controller/compute.py | 3 --- gns3server/utils/images.py | 17 +++++++++-------- tests/utils/test_images.py | 10 ++++++---- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/gns3server/compute/qemu/qemu_vm.py b/gns3server/compute/qemu/qemu_vm.py index 90381d6b..313a5c32 100644 --- a/gns3server/compute/qemu/qemu_vm.py +++ b/gns3server/compute/qemu/qemu_vm.py @@ -1057,10 +1057,10 @@ class QemuVM(BaseNode): # In case user upload image manually we don't have md5 sums. # We need generate hashes at this point, otherwise they will be generated # at asdict but not on separate thread. - await cancellable_wait_run_in_executor(md5sum, self._hda_disk_image) - await cancellable_wait_run_in_executor(md5sum, self._hdb_disk_image) - await cancellable_wait_run_in_executor(md5sum, self._hdc_disk_image) - await cancellable_wait_run_in_executor(md5sum, self._hdd_disk_image) + await cancellable_wait_run_in_executor(md5sum, self._hda_disk_image, self.working_dir) + await cancellable_wait_run_in_executor(md5sum, self._hdb_disk_image, self.working_dir) + await cancellable_wait_run_in_executor(md5sum, self._hdc_disk_image, self.working_dir) + await cancellable_wait_run_in_executor(md5sum, self._hdd_disk_image, self.working_dir) super().create() diff --git a/gns3server/controller/appliance_manager.py b/gns3server/controller/appliance_manager.py index b7874deb..17d8374a 100644 --- a/gns3server/controller/appliance_manager.py +++ b/gns3server/controller/appliance_manager.py @@ -153,8 +153,14 @@ class ApplianceManager: version_images[appliance_key] = image_in_db.filename else: # check if the image is on disk + # FIXME: still necessary? the image should have been discovered and saved in the db already image_path = os.path.join(image_dir, appliance_file) - if os.path.exists(image_path) and await wait_run_in_executor(md5sum, image_path) == image_checksum: + if os.path.exists(image_path) and \ + await wait_run_in_executor( + md5sum, + image_path, + cache_to_md5file=False + ) == image_checksum: async with aiofiles.open(image_path, "rb") as f: await write_image(appliance_file, image_path, f, images_repo) else: diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 0f19585b..d8caa17c 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -630,9 +630,6 @@ class Compute: try: if type in ["qemu", "dynamips", "iou"]: - # for local_image in list_images(type): - # if local_image['filename'] not in [i['filename'] for i in images]: - # images.append(local_image) images = sorted(images, key=itemgetter("filename")) else: images = sorted(images, key=itemgetter("image")) diff --git a/gns3server/utils/images.py b/gns3server/utils/images.py index 1ef85bc9..28f24a1a 100644 --- a/gns3server/utils/images.py +++ b/gns3server/utils/images.py @@ -122,7 +122,7 @@ async def read_image_info(path: str, expected_image_type: str = None) -> dict: "image_type": detected_image_type, "image_size": os.stat(path).st_size, "path": path, - "checksum": await wait_run_in_executor(md5sum, path), + "checksum": await wait_run_in_executor(md5sum, path, cache_to_md5file=False), "checksum_algorithm": "md5", } return image_info @@ -149,7 +149,7 @@ async def discover_images(image_type: str, skip_image_paths: list = None) -> Lis try: images.append(await read_image_info(path, image_type)) except InvalidImageError as e: - #log.warning(f"{e}") + log.debug(str(e)) continue return images @@ -211,7 +211,7 @@ def images_directories(image_type): return [force_unix_path(p) for p in paths if os.path.exists(p)] -def md5sum(path, working_dir=None, stopped_event=None): +def md5sum(path, working_dir=None, stopped_event=None, cache_to_md5file=True): """ Return the md5sum of an image and cache it on disk @@ -255,11 +255,12 @@ def md5sum(path, working_dir=None, stopped_event=None): log.error("Can't create digest of %s: %s", path, str(e)) return None - try: - with open(md5sum_file, "w+") as f: - f.write(digest) - except OSError as e: - log.error("Can't write digest of %s: %s", path, str(e)) + if cache_to_md5file: + try: + with open(md5sum_file, "w+") as f: + f.write(digest) + except OSError as e: + log.error("Can't write digest of %s: %s", path, str(e)) return digest diff --git a/tests/utils/test_images.py b/tests/utils/test_images.py index 5dd2270b..10c83521 100644 --- a/tests/utils/test_images.py +++ b/tests/utils/test_images.py @@ -18,6 +18,7 @@ import os import sys import threading +import pytest from unittest.mock import patch @@ -110,7 +111,8 @@ def test_remove_checksum(tmpdir): remove_checksum(str(tmpdir / 'not_exists')) -def test_list_images(tmpdir, config): +@pytest.mark.asyncio +async def test_list_images(tmpdir, config): path1 = tmpdir / "images1" / "IOS" / "test1.image" path1.write(b'\x7fELF\x01\x02\x01', ensure=True) @@ -140,7 +142,7 @@ def test_list_images(tmpdir, config): config.settings.Server.images_path = str(tmpdir / "images1") config.settings.Server.additional_images_paths = "/tmp/null24564;" + str(tmpdir / "images2") - assert list_images("dynamips") == [ + assert await list_images("dynamips") == [ { 'filename': 'test1.image', 'filesize': 7, @@ -156,7 +158,7 @@ def test_list_images(tmpdir, config): ] if sys.platform.startswith("linux"): - assert list_images("iou") == [ + assert await list_images("iou") == [ { 'filename': 'test3.bin', 'filesize': 7, @@ -165,7 +167,7 @@ def test_list_images(tmpdir, config): } ] - assert list_images("qemu") == [ + assert await list_images("qemu") == [ { 'filename': 'test4.qcow2', 'filesize': 1,