Compute a md5sum of images for futur purpose

Fix #234

gns3server/handlers/upload_handler.py

@@ -21,6 +21,7 @@ import stat
 
 from ..config import Config
 from ..web.route import Route
+from ..utils.images import remove_checksum, md5sum
 
 
 class UploadHandler:
@@ -36,7 +37,7 @@ class UploadHandler:
         try:
             for root, _, files in os.walk(UploadHandler.image_directory()):
                 for filename in files:
-                    if not filename.startswith("."):
+                    if not filename.startswith(".") and not filename.endswith(".md5sum"):
                         image_file = os.path.join(root, filename)
                         uploaded_files.append(image_file)
         except OSError:
@@ -70,12 +71,14 @@ class UploadHandler:
             destination_path = os.path.join(destination_dir, data["file"].filename)
         try:
             os.makedirs(destination_dir, exist_ok=True)
+            remove_checksum(destination_path)
             with open(destination_path, "wb+") as f:
                 while True:
                     chunk = data["file"].file.read(512)
                     if not chunk:
                         break
                     f.write(chunk)
+            md5sum(destination_path)
             st = os.stat(destination_path)
             os.chmod(destination_path, st.st_mode | stat.S_IXUSR)
         except OSError as e:

gns3server/modules/base_manager.py

@@ -37,6 +37,7 @@ from .nios.nio_udp import NIOUDP
 from .nios.nio_tap import NIOTAP
 from .nios.nio_nat import NIONAT
 from .nios.nio_generic_ethernet import NIOGenericEthernet
+from ..utils.images import md5sum, remove_checksum
 
 
 class BaseManager:
@@ -444,7 +445,7 @@ class BaseManager:
         files.sort()
         images = []
         for filename in files:
-            if filename[0] != ".":
+            if filename[0] != "." and not filename.endswith(".md5sum"):
                 images.append({"filename": filename})
         return images
 
@@ -461,6 +462,7 @@ class BaseManager:
         path = os.path.join(directory, os.path.basename(filename))
         log.info("Writting image file %s", path)
         try:
+            remove_checksum(path)
             os.makedirs(directory, exist_ok=True)
             with open(path, 'wb+') as f:
                 while True:
@@ -469,5 +471,6 @@ class BaseManager:
                         break
                     f.write(packet)
             os.chmod(path, stat.S_IWRITE | stat.S_IREAD | stat.S_IEXEC)
+            md5sum(path)
         except OSError as e:
             raise aiohttp.web.HTTPConflict(text="Could not write image: {} to {}".format(filename, e))

gns3server/modules/dynamips/nodes/router.py

@@ -37,6 +37,7 @@ from ..nios.nio_udp import NIOUDP
 
 from gns3server.config import Config
 from gns3server.utils.asyncio import wait_run_in_executor, monitor_process
+from gns3server.utils.images import md5sum
 
 
 class Router(BaseVM):
@@ -134,6 +135,7 @@ class Router(BaseVM):
                        "dynamips_id": self._dynamips_id,
                        "platform": self._platform,
                        "image": self._image,
+                       "image_md5sum": md5sum(self._image),
                        "startup_config": self._startup_config,
                        "private_config": self._private_config,
                        "ram": self._ram,

gns3server/modules/iou/iou_vm.py

@@ -46,6 +46,7 @@ from .utils.iou_import import nvram_import
 from .utils.iou_export import nvram_export
 from .ioucon import start_ioucon
 import gns3server.utils.asyncio
+import gns3server.utils.images
 
 
 import logging
@@ -208,6 +209,7 @@ class IOUVM(BaseVM):
                        "console": self._console,
                        "project_id": self.project.id,
                        "path": self.path,
+                       "md5sum": gns3server.utils.images.md5sum(self.path),
                        "ethernet_adapters": len(self._ethernet_adapters),
                        "serial_adapters": len(self._serial_adapters),
                        "ram": self._ram,
@@ -789,7 +791,7 @@ class IOUVM(BaseVM):
 
         # do not let IOU create the NVRAM anymore
         #startup_config_file = self.startup_config_file
-        #if startup_config_file:
+        # if startup_config_file:
         #    command.extend(["-c", os.path.basename(startup_config_file)])
 
         if self._l1_keepalives:

gns3server/modules/qemu/qemu_vm.py

@@ -38,6 +38,7 @@ from ..nios.nio_nat import NIONAT
 from ..base_vm import BaseVM
 from ...schemas.qemu import QEMU_OBJECT_SCHEMA, QEMU_PLATFORMS
 from ...utils.asyncio import monitor_process
+from ...utils.images import md5sum
 
 import logging
 log = logging.getLogger(__name__)
@@ -1217,13 +1218,23 @@ class QemuVM(BaseVM):
         # Qemu has a long list of options. The JSON schema is the single source of information
         for field in QEMU_OBJECT_SCHEMA["required"]:
             if field not in answer:
+                try:
                     answer[field] = getattr(self, field)
+                except AttributeError:
+                    pass
 
         answer["hda_disk_image"] = self.manager.get_relative_image_path(self._hda_disk_image)
+        answer["hda_disk_image_md5sum"] = md5sum(self._hda_disk_image)
         answer["hdb_disk_image"] = self.manager.get_relative_image_path(self._hdb_disk_image)
+        answer["hdb_disk_image_md5sum"] = md5sum(self._hdb_disk_image)
         answer["hdc_disk_image"] = self.manager.get_relative_image_path(self._hdc_disk_image)
+        answer["hdc_disk_image_md5sum"] = md5sum(self._hdc_disk_image)
         answer["hdd_disk_image"] = self.manager.get_relative_image_path(self._hdd_disk_image)
+        answer["hdd_disk_image_md5sum"] = md5sum(self._hdd_disk_image)
         answer["initrd"] = self.manager.get_relative_image_path(self._initrd)
+        answer["initrd_md5sum"] = md5sum(self._initrd)
+
         answer["kernel_image"] = self.manager.get_relative_image_path(self._kernel_image)
+        answer["kernel_image_md5sum"] = md5sum(self._kernel_image)
 
         return answer

gns3server/schemas/dynamips_vm.py

@@ -546,6 +546,11 @@ VM_OBJECT_SCHEMA = {
             "type": "string",
             "minLength": 1,
         },
+        "image_md5sum": {
+            "description": "checksum of the IOS image",
+            "type": "string",
+            "minLength": 1,
+        },
         "startup_config": {
             "description": "path to the IOS startup configuration file",
             "type": "string",

gns3server/schemas/iou.py

@@ -189,6 +189,10 @@ IOU_OBJECT_SCHEMA = {
             "description": "Path of iou binary",
             "type": "string"
         },
+        "md5sum": {
+            "description": "Checksum of iou binary",
+            "type": "string"
+        },
         "serial_adapters": {
             "description": "How many serial adapters are connected to the IOU",
             "type": "integer"
@@ -227,7 +231,7 @@ IOU_OBJECT_SCHEMA = {
         }
     },
     "additionalProperties": False,
-    "required": ["name", "vm_id", "console", "project_id", "path", "serial_adapters", "ethernet_adapters",
+    "required": ["name", "vm_id", "console", "project_id", "path", "md5sum", "serial_adapters", "ethernet_adapters",
                  "ram", "nvram", "l1_keepalives", "startup_config", "private_config", "use_default_iou_values"]
 }
 

gns3server/schemas/qemu.py

@@ -282,18 +282,34 @@ QEMU_OBJECT_SCHEMA = {
             "description": "QEMU hda disk image path",
             "type": "string",
         },
+        "hda_disk_image_md5sum": {
+            "description": "QEMU hda disk image checksum",
+            "type": ["string", "null"]
+        },
         "hdb_disk_image": {
             "description": "QEMU hdb disk image path",
             "type": "string",
         },
+        "hdb_disk_image_md5sum": {
+            "description": "QEMU hdb disk image checksum",
+            "type": ["string", "null"],
+        },
         "hdc_disk_image": {
             "description": "QEMU hdc disk image path",
             "type": "string",
         },
+        "hdc_disk_image_md5sum": {
+            "description": "QEMU hdc disk image checksum",
+            "type": ["string", "null"],
+        },
         "hdd_disk_image": {
             "description": "QEMU hdd disk image path",
             "type": "string",
         },
+        "hdd_disk_image_md5sum": {
+            "description": "QEMU hdd disk image checksum",
+            "type": ["string", "null"],
+        },
         "ram": {
             "description": "amount of RAM in MB",
             "type": "integer"
@@ -325,10 +341,18 @@ QEMU_OBJECT_SCHEMA = {
             "description": "QEMU initrd path",
             "type": "string",
         },
+        "initrd_md5sum": {
+            "description": "QEMU initrd path",
+            "type": ["string", "null"],
+        },
         "kernel_image": {
             "description": "QEMU kernel image path",
             "type": "string",
         },
+        "kernel_image_md5sum": {
+            "description": "QEMU kernel image checksum",
+            "type": ["string", "null"],
+        },
         "kernel_command_line": {
             "description": "QEMU kernel command line",
             "type": "string",
@@ -367,9 +391,10 @@ QEMU_OBJECT_SCHEMA = {
     },
     "additionalProperties": False,
     "required": ["vm_id", "project_id", "name", "qemu_path", "platform", "hda_disk_image", "hdb_disk_image",
-                 "hdc_disk_image", "hdd_disk_image", "ram", "adapters", "adapter_type", "mac_address", "console",
+                 "hdc_disk_image", "hdd_disk_image", "hda_disk_image_md5sum", "hdb_disk_image_md5sum",
-                 "initrd", "kernel_image", "kernel_command_line", "legacy_networking", "acpi_shutdown", "kvm",
+                 "hdc_disk_image_md5sum", "hdd_disk_image_md5sum", "ram", "adapters", "adapter_type", "mac_address",
-                 "cpu_throttling", "process_priority", "options"]
+                 "console", "initrd", "kernel_image", "initrd_md5sum", "kernel_image_md5sum", "kernel_command_line",
+                 "legacy_networking", "acpi_shutdown", "kvm", "cpu_throttling", "process_priority", "options"]
 }
 
 QEMU_BINARY_LIST_SCHEMA = {

gns3server/utils/images.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import hashlib
+
+
+def md5sum(path):
+    """
+    Return the md5sum of an image and cache it on disk
+
+    :param path: Path to the image
+    :returns: Digest of the image
+    """
+
+    if path is None or len(path) == 0:
+        return None
+
+    try:
+        with open(path + '.md5sum') as f:
+            return f.read()
+    except OSError:
+        pass
+
+    m = hashlib.md5()
+    with open(path, 'rb') as f:
+        while True:
+            buf = f.read(128)
+            if not buf:
+                break
+            m.update(buf)
+    digest = m.hexdigest()
+
+    try:
+        with open('{}.md5sum'.format(path), 'w+') as f:
+            f.write(digest)
+    except OSError as e:
+        log.error("Can't write digest of %s: %s", path, str(e))
+
+    return digest
+
+
+def remove_checksum(path):
+    """
+    Remove the checksum of an image from cache if exists
+    """
+
+    path = '{}.md5sum'.format(path)
+    if os.path.exists(path):
+        os.remove(path)

tests/handlers/api/test_dynamips.py

@@ -156,6 +156,10 @@ def test_upload_vm(server, tmpdir):
     with open(str(tmpdir / "test2")) as f:
         assert f.read() == "TEST"
 
+    with open(str(tmpdir / "test2.md5sum")) as f:
+        checksum = f.read()
+        assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf"
+
 
 def test_upload_vm_permission_denied(server, tmpdir):
     with open(str(tmpdir / "test2"), "w+") as f:

tests/handlers/api/test_iou.py

@@ -301,6 +301,7 @@ def test_get_configs_without_configs_file(server, vm):
     assert "startup_config" not in response.json
     assert "private_config" not in response.json
 
+
 def test_get_configs_with_startup_config_file(server, project, vm):
 
     path = startup_config_file(project, vm)
@@ -328,6 +329,10 @@ def test_upload_vm(server, tmpdir):
     with open(str(tmpdir / "test2")) as f:
         assert f.read() == "TEST"
 
+    with open(str(tmpdir / "test2.md5sum")) as f:
+        checksum = f.read()
+        assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf"
+
 
 def test_upload_vm_permission_denied(server, tmpdir):
     with open(str(tmpdir / "test2"), "w+") as f:

tests/handlers/api/test_qemu.py

@@ -88,10 +88,10 @@ def test_qemu_create_platform(server, project, base_params, fake_qemu_bin):
     assert response.json["platform"] == "x86_64"
 
 
-def test_qemu_create_with_params(server, project, base_params):
+def test_qemu_create_with_params(server, project, base_params, fake_qemu_vm):
     params = base_params
     params["ram"] = 1024
-    params["hda_disk_image"] = "/tmp/hda"
+    params["hda_disk_image"] = fake_qemu_vm
 
     response = server.post("/projects/{project_id}/qemu/vms".format(project_id=project.id), params, example=True)
     assert response.status == 201
@@ -99,7 +99,7 @@ def test_qemu_create_with_params(server, project, base_params):
     assert response.json["name"] == "PC TEST 1"
     assert response.json["project_id"] == project.id
     assert response.json["ram"] == 1024
-    assert response.json["hda_disk_image"] == "/tmp/hda"
+    assert response.json["hda_disk_image"] == fake_qemu_vm
 
 
 def test_qemu_get(server, project, vm):
@@ -152,18 +152,18 @@ def test_qemu_delete(server, vm):
         assert response.status == 204
 
 
-def test_qemu_update(server, vm, tmpdir, free_console_port, project):
+def test_qemu_update(server, vm, tmpdir, free_console_port, project, fake_qemu_vm):
     params = {
         "name": "test",
         "console": free_console_port,
         "ram": 1024,
-        "hdb_disk_image": "/tmp/hdb"
+        "hdb_disk_image": fake_qemu_vm
     }
     response = server.put("/projects/{project_id}/qemu/vms/{vm_id}".format(project_id=vm["project_id"], vm_id=vm["vm_id"]), params, example=True)
     assert response.status == 200
     assert response.json["name"] == "test"
     assert response.json["console"] == free_console_port
-    assert response.json["hdb_disk_image"] == "/tmp/hdb"
+    assert response.json["hdb_disk_image"] == fake_qemu_vm
     assert response.json["ram"] == 1024
 
 
@@ -225,6 +225,10 @@ def test_upload_vm(server, tmpdir):
     with open(str(tmpdir / "test2")) as f:
         assert f.read() == "TEST"
 
+    with open(str(tmpdir / "test2.md5sum")) as f:
+        checksum = f.read()
+        assert checksum == "033bd94b1168d7e4f0d644c3c95e35bf"
+
 
 def test_upload_vm_permission_denied(server, tmpdir):
     with open(str(tmpdir / "test2"), "w+") as f:

tests/handlers/test_upload.py

@@ -21,12 +21,23 @@ import os
 from unittest.mock import patch
 from gns3server.config import Config
 
-def test_index_upload(server):
+
+def test_index_upload(server, tmpdir):
+
+    Config.instance().set("Server", "images_path", str(tmpdir))
+
+    open(str(tmpdir / "alpha"), "w+").close()
+    open(str(tmpdir / "alpha.md5sum"), "w+").close()
+    open(str(tmpdir / ".beta"), "w+").close()
+
     response = server.get('/upload', api_version=None)
     assert response.status == 200
     html = response.html
     assert "GNS3 Server" in html
     assert "Select & Upload" in html
+    assert "alpha" in html
+    assert ".beta" not in html
+    assert "alpha.md5sum" not in html
 
 
 def test_upload(server, tmpdir):
@@ -40,9 +51,43 @@ def test_upload(server, tmpdir):
     body.add_field("file", open(str(tmpdir / "test"), "rb"), content_type="application/iou", filename="test2")
 
     Config.instance().set("Server", "images_path", str(tmpdir))
+
     response = server.post('/upload', api_version=None, body=body, raw=True)
 
+    assert "test2" in response.body.decode("utf-8")
+
     with open(str(tmpdir / "QEMU" / "test2")) as f:
         assert f.read() == content
 
+    with open(str(tmpdir / "QEMU" / "test2.md5sum")) as f:
+        checksum = f.read()
+        assert checksum == "ae187e1febee2a150b64849c32d566ca"
+
+
+def test_upload_previous_checksum(server, tmpdir):
+
+    content = ''.join(['a' for _ in range(0, 1025)])
+
+    with open(str(tmpdir / "test"), "w+") as f:
+        f.write(content)
+    body = aiohttp.FormData()
+    body.add_field("type", "QEMU")
+    body.add_field("file", open(str(tmpdir / "test"), "rb"), content_type="application/iou", filename="test2")
+
+    Config.instance().set("Server", "images_path", str(tmpdir))
+
+    os.makedirs(str(tmpdir / "QEMU"))
+
+    with open(str(tmpdir / "QEMU" / "test2.md5sum"), 'w+') as f:
+        f.write("FAKE checksum")
+
+    response = server.post('/upload', api_version=None, body=body, raw=True)
+
     assert "test2" in response.body.decode("utf-8")
+
+    with open(str(tmpdir / "QEMU" / "test2")) as f:
+        assert f.read() == content
+
+    with open(str(tmpdir / "QEMU" / "test2.md5sum")) as f:
+        checksum = f.read()
+        assert checksum == "ae187e1febee2a150b64849c32d566ca"

tests/modules/test_manager.py

@@ -125,7 +125,7 @@ def test_get_relative_image_path(qemu, tmpdir):
 
 def test_list_images(loop, qemu, tmpdir):
 
-    fake_images = ["a.bin", "b.bin", ".blu.bin"]
+    fake_images = ["a.bin", "b.bin", ".blu.bin", "a.bin.md5sum"]
     for image in fake_images:
         with open(str(tmpdir / image), "w+") as f:
             f.write("1")

tests/utils/test_images.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2014 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from gns3server.utils.images import md5sum, remove_checksum
+
+
+def test_md5sum(tmpdir):
+    fake_img = str(tmpdir / 'hello')
+
+    with open(fake_img, 'w+') as f:
+        f.write('hello')
+
+    assert md5sum(fake_img) == '5d41402abc4b2a76b9719d911017c592'
+    with open(str(tmpdir / 'hello.md5sum')) as f:
+        assert f.read() == '5d41402abc4b2a76b9719d911017c592'
+
+
+def test_md5sum_existing_digest(tmpdir):
+    fake_img = str(tmpdir / 'hello')
+
+    with open(str(tmpdir / 'hello.md5sum'), 'w+') as f:
+        f.write('aaaaa02abc4b2a76b9719d911017c592')
+
+    assert md5sum(fake_img) == 'aaaaa02abc4b2a76b9719d911017c592'
+
+
+def test_md5sum_none(tmpdir):
+    assert md5sum(None) is None
+
+
+def test_remove_checksum(tmpdir):
+
+    with open(str(tmpdir / 'hello.md5sum'), 'w+') as f:
+        f.write('aaaaa02abc4b2a76b9719d911017c592')
+    remove_checksum(str(tmpdir / 'hello'))
+
+    assert not os.path.exists(str(tmpdir / 'hello.md5sum'))
+
+    remove_checksum(str(tmpdir / 'not_exists'))