Merge pull request #2448 from GNS3/bugfix/3664

Fix listing images

gns3server/utils/images.py

@@ -45,7 +45,7 @@ def list_images(emulator_type):
 
         # We limit recursion to path outside the default images directory
         # the reason is in the default directory manage file organization and
-        # it should be flatten to keep things simple
+        # it should be flat to keep things simple
         recurse = True
         if os.path.commonprefix([directory, general_images_directory]) == general_images_directory:
             recurse = False
@@ -53,37 +53,53 @@ def list_images(emulator_type):
         directory = os.path.normpath(directory)
         for root, _, filenames in _os_walk(directory, recurse=recurse):
             for filename in filenames:
-                if filename not in files:
+                if filename in files:
-                    if filename.endswith(".md5sum") or filename.startswith("."):
+                    log.debug("File {} has already been found, skipping...".format(filename))
+                    continue
+                if filename.endswith(".md5sum") or filename.startswith("."):
+                    continue
+
+                files.add(filename)
+
+                filesize = os.stat(os.path.join(root, filename)).st_size
+                if filesize < 7:
+                    log.debug("File {} is too small to be an image, skipping...".format(filename))
+                    continue
+
+                try:
+                    with open(os.path.join(root, filename), "rb") as f:
+                        # read the first 7 bytes of the file.
+                        elf_header_start = f.read(7)
+                    if emulator_type == "dynamips" and elf_header_start != b'\x7fELF\x01\x02\x01':
+                        # IOS images must start with the ELF magic number, be 32-bit, big endian and have an ELF version of 1
+                        log.warning("IOS image {} does not start with a valid ELF magic number, skipping...".format(filename))
+                        continue
+                    elif emulator_type == "iou" and elf_header_start != b'\x7fELF\x02\x01\x01' and elf_header_start != b'\x7fELF\x01\x01\x01':
+                        # IOU images must start with the ELF magic number, be 32-bit or 64-bit, little endian and have an ELF version of 1
+                        log.warning("IOU image {} does not start with a valid ELF magic number, skipping...".format(filename))
+                        continue
+                    elif emulator_type == "qemu" and elf_header_start[:4] == b'\x7fELF':
+                        # QEMU images should not start with an ELF magic number
+                        log.warning("QEMU image {} starts with an ELF magic number, skipping...".format(filename))
                         continue
-                    elif ((filename.endswith(".image") or filename.endswith(".bin")) and emulator_type == "dynamips") \
-                            or ((filename.endswith(".bin") or filename.startswith("i86bi")) and emulator_type == "iou") \
-                            or (not filename.endswith(".bin") and not filename.endswith(".image") and emulator_type == "qemu"):
-                        files.add(filename)
 
-                        # It the image is located in the standard directory the path is relative
+                    # It the image is located in the standard directory the path is relative
-                        if os.path.commonprefix([root, default_directory]) != default_directory:
+                    if os.path.commonprefix([root, default_directory]) != default_directory:
-                            path = os.path.join(root, filename)
+                        path = os.path.join(root, filename)
-                        else:
+                    else:
-                            path = os.path.relpath(os.path.join(root, filename), default_directory)
+                        path = os.path.relpath(os.path.join(root, filename), default_directory)
 
-                        try:
+                    images.append(
-                            if emulator_type in ["dynamips", "iou"]:
+                        {
-                                with open(os.path.join(root, filename), "rb") as f:
+                            "filename": filename,
-                                    # read the first 7 bytes of the file.
+                            "path": force_unix_path(path),
-                                    elf_header_start = f.read(7)
+                            "md5sum": md5sum(os.path.join(root, filename)),
-                                # valid IOU or IOS images must start with the ELF magic number, be 32-bit or 64-bit,
+                            "filesize": filesize
-                                # little endian and have an ELF version of 1
+                         }
-                                if elf_header_start != b'\x7fELF\x02\x01\x01' and elf_header_start != b'\x7fELF\x01\x01\x01':
+                    )
-                                    continue
 
-                            images.append({
+                except OSError as e:
-                                "filename": filename,
+                    log.warning("Can't add image {}: {}".format(path, str(e)))
-                                "path": force_unix_path(path),
-                                "md5sum": md5sum(os.path.join(root, filename)),
-                                "filesize": os.stat(os.path.join(root, filename)).st_size})
-                        except OSError as e:
-                            log.warning("Can't add image {}: {}".format(path, str(e)))
     return images
 
 

tests/compute/test_manager.py

@@ -239,12 +239,12 @@ async def test_list_images(qemu, tmpdir):
     os.makedirs(tmp_images_dir, exist_ok=True)
     for image in fake_images:
         with open(os.path.join(tmp_images_dir, image), "w+") as f:
-            f.write("1")
+            f.write("1234567")
 
     with patch("gns3server.utils.images.default_images_directory", return_value=str(tmp_images_dir)):
         assert sorted(await qemu.list_images(), key=lambda k: k['filename']) == [
-            {"filename": "a.qcow2", "path": "a.qcow2", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1},
+            {"filename": "a.qcow2", "path": "a.qcow2", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7},
-            {"filename": "b.qcow2", "path": "b.qcow2", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}
+            {"filename": "b.qcow2", "path": "b.qcow2", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7}
         ]
 
 
@@ -255,19 +255,19 @@ async def test_list_images_recursives(qemu, tmpdir):
     fake_images = ["a.qcow2", "b.qcow2", ".blu.qcow2", "a.qcow2.md5sum"]
     for image in fake_images:
         with open(os.path.join(tmp_images_dir, image), "w+") as f:
-            f.write("1")
+            f.write("1234567")
     os.makedirs(os.path.join(tmp_images_dir, "c"))
     fake_images = ["c.qcow2", "c.qcow2.md5sum"]
     for image in fake_images:
         with open(os.path.join(tmp_images_dir, "c", image), "w+") as f:
-            f.write("1")
+            f.write("1234567")
 
     with patch("gns3server.utils.images.default_images_directory", return_value=str(tmp_images_dir)):
 
         assert sorted(await qemu.list_images(), key=lambda k: k['filename']) == [
-            {"filename": "a.qcow2", "path": "a.qcow2", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1},
+            {"filename": "a.qcow2", "path": "a.qcow2", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7},
-            {"filename": "b.qcow2", "path": "b.qcow2", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1},
+            {"filename": "b.qcow2", "path": "b.qcow2", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7},
-            {"filename": "c.qcow2", "path": force_unix_path(os.path.sep.join(["c", "c.qcow2"])), "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1}
+            {"filename": "c.qcow2", "path": force_unix_path(os.path.sep.join(["c", "c.qcow2"])), "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7}
         ]
 
 

tests/handlers/api/compute/test_dynamips.py

@@ -144,7 +144,7 @@ def fake_image(tmpdir):
 
     path = str(tmpdir / "7200.bin")
     with open(path, "wb+") as f:
-        f.write(b'\x7fELF\x01\x01\x01')
+        f.write(b'\x7fELF\x01\x02\x01')
     os.chmod(path, stat.S_IREAD)
     return path
 
@@ -168,7 +168,7 @@ async def test_images(compute_api, tmpdir, fake_image, fake_file):
     assert response.json == [{"filename": "7200.bin",
                               "path": "7200.bin",
                               "filesize": 7,
-                              "md5sum": "e573e8f5c93c6c00783f20c7a170aa6c"
+                              "md5sum": "b0d5aa897d937aced5a6b1046e8f7e2e"
                               }]
 
 

tests/handlers/api/compute/test_qemu.py

@@ -45,7 +45,7 @@ def fake_qemu_vm(images_dir):
     img_dir = os.path.join(images_dir, "QEMU")
     bin_path = os.path.join(img_dir, "linux载.img")
     with open(bin_path, "w+") as f:
-        f.write("1")
+        f.write("1234567")
     os.chmod(bin_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
     return bin_path
 
@@ -101,7 +101,7 @@ async def test_qemu_create_with_params(compute_api, compute_project, base_params
     assert response.json["project_id"] == compute_project.id
     assert response.json["ram"] == 1024
     assert response.json["hda_disk_image"] == "linux载.img"
-    assert response.json["hda_disk_image_md5sum"] == "c4ca4238a0b923820dcc509a6f75849b"
+    assert response.json["hda_disk_image_md5sum"] == "fcea920f7412b5da7be0cf42b8c93759"
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Not supported on Windows")
@@ -279,7 +279,7 @@ async def test_images(compute_api, fake_qemu_vm):
 
     response = await compute_api.get("/qemu/images")
     assert response.status == 200
-    assert {"filename": "linux载.img", "path": "linux载.img", "md5sum": "c4ca4238a0b923820dcc509a6f75849b", "filesize": 1} in response.json
+    assert {"filename": "linux载.img", "path": "linux载.img", "md5sum": "fcea920f7412b5da7be0cf42b8c93759", "filesize": 7} in response.json
 
 
 @pytest.mark.skipif(sys.platform.startswith("win"), reason="Does not work on Windows")

tests/utils/test_images.py

@@ -114,30 +114,49 @@ def test_remove_checksum(tmpdir):
 
 def test_list_images(tmpdir):
 
-    path1 = tmpdir / "images1" / "IOS" / "test1.image"
+    # IOS image in the images directory
-    path1.write(b'\x7fELF\x01\x01\x01', ensure=True)
+    ios_image_1 = tmpdir / "images1" / "IOS" / "ios_image_1.image"
-    path1 = force_unix_path(str(path1))
+    ios_image_1.write(b'\x7fELF\x01\x02\x01', ensure=True)
+    ios_image_1 = force_unix_path(str(ios_image_1))
 
-    path2 = tmpdir / "images2" / "test2.image"
+    # IOS image in an additional images path
-    path2.write(b'\x7fELF\x01\x01\x01', ensure=True)
+    ios_image_2 = tmpdir / "images2" / "ios_image_2.image"
-    path2 = force_unix_path(str(path2))
+    ios_image_2.write(b'\x7fELF\x01\x02\x01', ensure=True)
+    ios_image_2 = force_unix_path(str(ios_image_2))
 
-    # Invalid image because not a valid elf file
+    # Not a valid elf file
-    path = tmpdir / "images2" / "test_invalid.image"
+    not_elf_file = tmpdir / "images1" / "IOS" / "not_elf.image"
-    path.write(b'NOTANELF', ensure=True)
+    not_elf_file.write(b'NOTANELF', ensure=True)
+    not_elf_file = force_unix_path(str(not_elf_file))
+
+    # Invalid image because it is very small
+    small_file = tmpdir / "images1" / "too_small.image"
+    small_file.write(b'1', ensure=True)
 
     if sys.platform.startswith("linux"):
-        path3 = tmpdir / "images1" / "IOU" / "test3.bin"
+        # 64-bit IOU image
-        path3.write(b'\x7fELF\x02\x01\x01', ensure=True)
+        iou_image_1 = tmpdir / "images1" / "IOU" / "iou64.bin"
-        path3 = force_unix_path(str(path3))
+        iou_image_1.write(b'\x7fELF\x02\x01\x01', ensure=True)
+        iou_image_1 = force_unix_path(str(iou_image_1))
+        # 32-bit IOU image
+        iou_image_2 = tmpdir / "images1" / "IOU" / "iou32.bin"
+        iou_image_2.write(b'\x7fELF\x01\x01\x01', ensure=True) # 32-bit IOU image
+        iou_image_2 = force_unix_path(str(iou_image_2))
 
-    path4 = tmpdir / "images1" / "QEMU" / "test4.qcow2"
-    path4.write("1", ensure=True)
-    path4 = force_unix_path(str(path4))
 
-    path5 = tmpdir / "images1" / "QEMU" / "test4.qcow2.md5sum"
+    # Qemu image
-    path5.write("1", ensure=True)
+    qemu_image_1 = tmpdir / "images1" / "QEMU" / "qemu_image.qcow2"
-    path5 = force_unix_path(str(path5))
+    qemu_image_1.write("1234567", ensure=True)
+    qemu_image_1 = force_unix_path(str(qemu_image_1))
+
+    # ELF file inside the Qemu
+    elf_file = tmpdir / "images1" / "QEMU" / "elf_file.bin"
+    elf_file.write(b'\x7fELF\x02\x01\x01', ensure=True)  # ELF file
+    elf_file = force_unix_path(str(elf_file))
+
+    md5sum_file = tmpdir / "images1" / "QEMU" / "image.qcow2.md5sum"
+    md5sum_file.write("1", ensure=True)
+    md5sum_file = force_unix_path(str(md5sum_file))
 
     with patch("gns3server.config.Config.get_section_config", return_value={
             "images_path": str(tmpdir / "images1"),
@@ -146,34 +165,40 @@ def test_list_images(tmpdir):
 
         assert list_images("dynamips") == [
             {
-                'filename': 'test1.image',
+                'filename': 'ios_image_1.image',
                 'filesize': 7,
-                'md5sum': 'e573e8f5c93c6c00783f20c7a170aa6c',
+                'md5sum': 'b0d5aa897d937aced5a6b1046e8f7e2e',
-                'path': 'test1.image'
+                'path': 'ios_image_1.image'
             },
             {
-                'filename': 'test2.image',
+                'filename': 'ios_image_2.image',
                 'filesize': 7,
-                'md5sum': 'e573e8f5c93c6c00783f20c7a170aa6c',
+                'md5sum': 'b0d5aa897d937aced5a6b1046e8f7e2e',
-                'path': str(path2)
+                'path': str(ios_image_2)
             }
         ]
 
         if sys.platform.startswith("linux"):
             assert list_images("iou") == [
                 {
-                    'filename': 'test3.bin',
+                    'filename': 'iou64.bin',
                     'filesize': 7,
                     'md5sum': 'c73626d23469519894d58bc98bee9655',
-                    'path': 'test3.bin'
+                    'path': 'iou64.bin'
+                },
+                {
+                    'filename': 'iou32.bin',
+                    'filesize': 7,
+                    'md5sum': 'e573e8f5c93c6c00783f20c7a170aa6c',
+                    'path': 'iou32.bin'
                 }
             ]
 
         assert list_images("qemu") == [
             {
-                'filename': 'test4.qcow2',
+                'filename': 'qemu_image.qcow2',
-                'filesize': 1,
+                'filesize': 7,
-                'md5sum': 'c4ca4238a0b923820dcc509a6f75849b',
+                'md5sum': 'fcea920f7412b5da7be0cf42b8c93759',
-                'path': 'test4.qcow2'
+                'path': 'qemu_image.qcow2'
             }
         ]