Merge pull request #478 from GNS3/qcow2_rebase

Qcow2 rebase

gns3server/modules/qemu/qcow2.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2016 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import asyncio
+import struct
+
+
+class Qcow2Error(Exception):
+    pass
+
+
+class Qcow2:
+    """
+    Allow to parse a Qcow2 file
+    """
+
+    def __init__(self, path):
+
+        self.path = path
+        self._reload()
+
+    def _reload(self):
+        # Each QCOW2 file begins with a header, in big endian format, as follows:
+        #
+        # typedef struct QCowHeader {
+        #     uint32_t magic;
+        #     uint32_t version;
+        #
+        #     uint64_t backing_file_offset;
+        #     uint32_t backing_file_size;
+        #
+        #     uint32_t cluster_bits;
+        #     uint64_t size; /* in bytes */
+        #     uint32_t crypt_method;
+        #
+        #     uint32_t l1_size;
+        #     uint64_t l1_table_offset;
+        #
+        #     uint64_t refcount_table_offset;
+        #     uint32_t refcount_table_clusters;
+        #
+        #     uint32_t nb_snapshots;
+        #     uint64_t snapshots_offset;
+        # } QCowHeader;
+        struct_format = ">IIQi"
+
+        with open(self.path, 'rb') as f:
+            content = f.read(struct.calcsize(struct_format))
+
+            self.magic, self.version, self.backing_file_offset, self.backing_file_size = struct.unpack_from(struct_format, content)
+
+        if self.magic != 1363560955:  # The first 4 bytes contain the characters 'Q', 'F', 'I' followed by 0xfb.
+            raise Qcow2Error("Invalid magic for {}".format(self.path))
+
+    @property
+    def backing_file(self):
+        """
+        When using linked clone this will return the path to the base image
+
+        :returns: None if it's not a linked clone, the path otherwise
+        """
+        with open(self.path, 'rb') as f:
+            f.seek(self.backing_file_offset)
+            content = f.read(self.backing_file_size)
+        path = content.decode()
+        if len(path) == 0:
+            return None
+        return path
+
+    @asyncio.coroutine
+    def rebase(self, qemu_img, base_image):
+        """
+        Rebase a linked clone in order to use the correct disk
+
+        :param qemu_img: Path to the qemu-img binary
+        :param base_image: Path to the base image
+        """
+        if not os.path.exists(base_image):
+            raise FileNotFoundError(base_image)
+        command = [qemu_img, "rebase", "-u", "-b", base_image, self.path]
+        process = yield from asyncio.create_subprocess_exec(*command)
+        retcode = yield from process.wait()
+        if retcode != 0:
+            raise Qcow2Error("Could not rebase the image")
+        self._reload()

gns3server/modules/qemu/qemu_vm.py

@@ -40,6 +40,7 @@ from ..base_vm import BaseVM
 from ...schemas.qemu import QEMU_OBJECT_SCHEMA, QEMU_PLATFORMS
 from ...utils.asyncio import monitor_process
 from ...utils.images import md5sum
+from .qcow2 import Qcow2, Qcow2Error
 
 import logging
 log = logging.getLogger(__name__)
@@ -1233,90 +1234,45 @@ class QemuVM(BaseVM):
         options = []
         qemu_img_path = self._get_qemu_img()
 
-        if self._hda_disk_image:
+        drives = ["a", "b", "c", "d"]
-            if not os.path.isfile(self._hda_disk_image) or not os.path.exists(self._hda_disk_image):
+
-                if os.path.islink(self._hda_disk_image):
+        for disk_index, drive in enumerate(drives):
-                    raise QemuError("hda disk image '{}' linked to '{}' is not accessible".format(self._hda_disk_image, os.path.realpath(self._hda_disk_image)))
+            disk_image = getattr(self, "_hd{}_disk_image".format(drive))
+            interface = getattr(self, "hd{}_disk_interface".format(drive))
+
+            if not disk_image:
+                continue
+
+            disk_name = "hd" + drive
+
+            if not os.path.isfile(disk_image) or not os.path.exists(disk_image):
+                if os.path.islink(disk_image):
+                    raise QemuError("{} disk image '{}' linked to '{}' is not accessible".format(disk_name, disk_image, os.path.realpath(disk_image)))
                 else:
-                    raise QemuError("hda disk image '{}' is not accessible".format(self._hda_disk_image))
+                    raise QemuError("{} disk image '{}' is not accessible".format(disk_name, disk_image))
             if self._linked_clone:
-                hda_disk = os.path.join(self.working_dir, "hda_disk.qcow2")
+                disk = os.path.join(self.working_dir, "{}_disk.qcow2".format(disk_name))
-                if not os.path.exists(hda_disk):
+                if not os.path.exists(disk):
                     # create the disk
                     try:
                         process = yield from asyncio.create_subprocess_exec(qemu_img_path, "create", "-o",
-                                                                            "backing_file={}".format(self._hda_disk_image),
+                                                                            "backing_file={}".format(disk_image),
-                                                                            "-f", "qcow2", hda_disk)
+                                                                            "-f", "qcow2", disk)
                         retcode = yield from process.wait()
                         log.info("{} returned with {}".format(qemu_img_path, retcode))
                     except (OSError, subprocess.SubprocessError) as e:
-                        raise QemuError("Could not create hda disk image {}".format(e))
+                        raise QemuError("Could not create {} disk image {}".format(disk_name, e))
-            else:
-                hda_disk = self._hda_disk_image
-            options.extend(["-drive", 'file={},if={},index=0,media=disk'.format(hda_disk, self.hda_disk_interface)])
-
-        if self._hdb_disk_image:
-            if not os.path.isfile(self._hdb_disk_image) or not os.path.exists(self._hdb_disk_image):
-                if os.path.islink(self._hdb_disk_image):
-                    raise QemuError("hdb disk image '{}' linked to '{}' is not accessible".format(self._hdb_disk_image, os.path.realpath(self._hdb_disk_image)))
                 else:
-                    raise QemuError("hdb disk image '{}' is not accessible".format(self._hdb_disk_image))
+                    # The disk exists we check if the clone work
-            if self._linked_clone:
-                hdb_disk = os.path.join(self.working_dir, "hdb_disk.qcow2")
-                if not os.path.exists(hdb_disk):
                     try:
-                        process = yield from asyncio.create_subprocess_exec(qemu_img_path, "create", "-o",
+                        qcow2 = Qcow2(disk)
-                                                                            "backing_file={}".format(self._hdb_disk_image),
+                        yield from qcow2.rebase(qemu_img_path, disk_image)
-                                                                            "-f", "qcow2", hdb_disk)
+                    except (Qcow2Error, OSError) as e:
-                        retcode = yield from process.wait()
+                        raise QemuError("Could not use qcow2 disk image {} for {} {}".format(disk_image, disk_name, e))
-                        log.info("{} returned with {}".format(qemu_img_path, retcode))
-                    except (OSError, subprocess.SubprocessError) as e:
-                        raise QemuError("Could not create hdb disk image {}".format(e))
-            else:
-                hdb_disk = self._hdb_disk_image
-            options.extend(["-drive", 'file={},if={},index=1,media=disk'.format(hdb_disk, self.hdb_disk_interface)])
 
-        if self._hdc_disk_image:
-            if not os.path.isfile(self._hdc_disk_image) or not os.path.exists(self._hdc_disk_image):
-                if os.path.islink(self._hdc_disk_image):
-                    raise QemuError("hdc disk image '{}' linked to '{}' is not accessible".format(self._hdc_disk_image, os.path.realpath(self._hdc_disk_image)))
-                else:
-                    raise QemuError("hdc disk image '{}' is not accessible".format(self._hdc_disk_image))
-            if self._linked_clone:
-                hdc_disk = os.path.join(self.working_dir, "hdc_disk.qcow2")
-                if not os.path.exists(hdc_disk):
-                    try:
-                        process = yield from asyncio.create_subprocess_exec(qemu_img_path, "create", "-o",
-                                                                            "backing_file={}".format(self._hdc_disk_image),
-                                                                            "-f", "qcow2", hdc_disk)
-                        retcode = yield from process.wait()
-                        log.info("{} returned with {}".format(qemu_img_path, retcode))
-                    except (OSError, subprocess.SubprocessError) as e:
-                        raise QemuError("Could not create hdc disk image {}".format(e))
             else:
-                hdc_disk = self._hdc_disk_image
+                disk = disk_image
-            options.extend(["-drive", 'file={},if={},index=2,media=disk'.format(hdc_disk, self.hdc_disk_interface)])
+            options.extend(["-drive", 'file={},if={},index={},media=disk'.format(disk, interface, disk_index)])
-
-        if self._hdd_disk_image:
-            if not os.path.isfile(self._hdd_disk_image) or not os.path.exists(self._hdd_disk_image):
-                if os.path.islink(self._hdd_disk_image):
-                    raise QemuError("hdd disk image '{}' linked to '{}' is not accessible".format(self._hdd_disk_image, os.path.realpath(self._hdd_disk_image)))
-                else:
-                    raise QemuError("hdd disk image '{}' is not accessible".format(self._hdd_disk_image))
-            if self._linked_clone:
-                hdd_disk = os.path.join(self.working_dir, "hdd_disk.qcow2")
-                if not os.path.exists(hdd_disk):
-                    try:
-                        process = yield from asyncio.create_subprocess_exec(qemu_img_path, "create", "-o",
-                                                                            "backing_file={}".format(self._hdd_disk_image),
-                                                                            "-f", "qcow2", hdd_disk)
-                        retcode = yield from process.wait()
-                        log.info("{} returned with {}".format(qemu_img_path, retcode))
-                    except (OSError, subprocess.SubprocessError) as e:
-                        raise QemuError("Could not create hdd disk image {}".format(e))
-            else:
-                hdd_disk = self._hdd_disk_image
-            options.extend(["-drive", 'file={},if={},index=3,media=disk'.format(hdd_disk, self.hdd_disk_interface)])
 
         return options
 

tests/modules/qemu/test_qcow2.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2016 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import pytest
+import shutil
+import asyncio
+
+from gns3server.modules.qemu.qcow2 import Qcow2, Qcow2Error
+
+
+def qemu_img():
+    """
+    Return the path of qemu-img on system.
+    We can't use shutil.which because for safety reason we break
+    the PATH to avoid test interacting with real binaries
+    """
+    paths = [
+        "/usr/bin/qemu-img",
+        "/usr/local/bin/qemu-img"
+    ]
+    for path in paths:
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def test_valid_base_file():
+    qcow2 = Qcow2("tests/resources/empty8G.qcow2")
+    assert qcow2.version == 3
+    assert qcow2.backing_file is None
+
+
+def test_valid_linked_file():
+    qcow2 = Qcow2("tests/resources/linked.qcow2")
+    assert qcow2.version == 3
+    assert qcow2.backing_file == "empty8G.qcow2"
+
+
+def test_invalid_file():
+    with pytest.raises(Qcow2Error):
+        Qcow2("tests/resources/nvram_iou")
+
+
+@pytest.mark.skipif(qemu_img() is None, reason="qemu-img is not available")
+def test_rebase(tmpdir, loop):
+    shutil.copy("tests/resources/empty8G.qcow2", str(tmpdir / "empty16G.qcow2"))
+    shutil.copy("tests/resources/linked.qcow2", str(tmpdir / "linked.qcow2"))
+    qcow2 = Qcow2(str(tmpdir / "linked.qcow2"))
+    assert qcow2.version == 3
+    assert qcow2.backing_file == "empty8G.qcow2"
+    loop.run_until_complete(asyncio.async(qcow2.rebase(qemu_img(), str(tmpdir / "empty16G.qcow2"))))
+    assert qcow2.backing_file == str(tmpdir / "empty16G.qcow2")

tests/modules/qemu/test_qemu_vm.py

@@ -321,11 +321,35 @@ def test_disk_options(vm, tmpdir, loop, fake_qemu_img_binary):
     open(vm._hda_disk_image, "w+").close()
 
     with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process:
-        loop.run_until_complete(asyncio.async(vm._disk_options()))
+        options = loop.run_until_complete(asyncio.async(vm._disk_options()))
         assert process.called
         args, kwargs = process.call_args
         assert args == (fake_qemu_img_binary, "create", "-o", "backing_file={}".format(vm._hda_disk_image), "-f", "qcow2", os.path.join(vm.working_dir, "hda_disk.qcow2"))
 
+    assert options == ['-drive', 'file=' + os.path.join(vm.working_dir, "hda_disk.qcow2") + ',if=ide,index=0,media=disk']
+
+
+def test_disk_options_multiple_disk(vm, tmpdir, loop, fake_qemu_img_binary):
+
+    vm._hda_disk_image = str(tmpdir / "test0.qcow2")
+    vm._hdb_disk_image = str(tmpdir / "test1.qcow2")
+    vm._hdc_disk_image = str(tmpdir / "test2.qcow2")
+    vm._hdd_disk_image = str(tmpdir / "test3.qcow2")
+    open(vm._hda_disk_image, "w+").close()
+    open(vm._hdb_disk_image, "w+").close()
+    open(vm._hdc_disk_image, "w+").close()
+    open(vm._hdd_disk_image, "w+").close()
+
+    with asyncio_patch("asyncio.create_subprocess_exec", return_value=MagicMock()) as process:
+        options = loop.run_until_complete(asyncio.async(vm._disk_options()))
+
+    assert options == [
+        '-drive', 'file=' + os.path.join(vm.working_dir, "hda_disk.qcow2") + ',if=ide,index=0,media=disk',
+        '-drive', 'file=' + os.path.join(vm.working_dir, "hdb_disk.qcow2") + ',if=ide,index=1,media=disk',
+        '-drive', 'file=' + os.path.join(vm.working_dir, "hdc_disk.qcow2") + ',if=ide,index=2,media=disk',
+        '-drive', 'file=' + os.path.join(vm.working_dir, "hdd_disk.qcow2") + ',if=ide,index=3,media=disk'
+    ]
+
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Not supported on Windows")
 def test_set_process_priority(vm, loop, fake_qemu_img_binary):