Finalize image management refactoring and auto install appliance if possible

gns3server/api/routes/compute/qemu_nodes.py

@@ -139,13 +139,6 @@ async def start_qemu_node(node: QemuVM = Depends(dep_node)) -> Response:
     Start a Qemu node.
     """
 
-    qemu_manager = Qemu.instance()
-    hardware_accel = qemu_manager.config.settings.Qemu.enable_hardware_acceleration
-    if hardware_accel and "-machine accel=tcg" not in node.options:
-        pm = ProjectManager.instance()
-        if pm.check_hardware_virtualization(node) is False:
-            pass  # FIXME: check this
-            # raise ComputeError("Cannot start VM with hardware acceleration (KVM/HAX) enabled because hardware virtualization (VT-x/AMD-V) is already used by another software like VMware or VirtualBox")
     await node.start()
     return Response(status_code=status.HTTP_204_NO_CONTENT)
 

gns3server/api/routes/controller/images.py

@@ -26,9 +26,14 @@ from fastapi import APIRouter, Request, Response, Depends, status
 from sqlalchemy.orm.exc import MultipleResultsFound
 from typing import List
 from gns3server import schemas
+from pydantic import ValidationError
 
 from gns3server.utils.images import InvalidImageError, default_images_directory, write_image
 from gns3server.db.repositories.images import ImagesRepository
+from gns3server.db.repositories.templates import TemplatesRepository
+from gns3server.services.templates import TemplatesService
+from gns3server.db.repositories.rbac import RbacRepository
+from gns3server.controller import Controller
 from gns3server.controller.controller_error import (
     ControllerError,
     ControllerNotFoundError,
@@ -36,6 +41,7 @@ from gns3server.controller.controller_error import (
     ControllerBadRequestError
 )
 
+from .dependencies.authentication import get_current_active_user
 from .dependencies.database import get_repository
 
 log = logging.getLogger(__name__)
@@ -43,7 +49,7 @@ log = logging.getLogger(__name__)
 router = APIRouter()
 
 
-@router.get("")
+@router.get("", response_model=List[schemas.Image])
 async def get_images(
         images_repo: ImagesRepository = Depends(get_repository(ImagesRepository)),
 ) -> List[schemas.Image]:
@@ -60,9 +66,15 @@ async def upload_image(
         request: Request,
         image_type: schemas.ImageType = schemas.ImageType.qemu,
         images_repo: ImagesRepository = Depends(get_repository(ImagesRepository)),
+        templates_repo: TemplatesRepository = Depends(get_repository(TemplatesRepository)),
+        current_user: schemas.User = Depends(get_current_active_user),
+        rbac_repo: RbacRepository = Depends(get_repository(RbacRepository))
 ) -> schemas.Image:
     """
     Upload an image.
+
+    Example: curl -X POST http://host:port/v3/images/upload/my_image_name.qcow2?image_type=qemu \
+    -H 'Authorization: Bearer <token>' --data-binary @"/path/to/image.qcow2"
     """
 
     image_path = urllib.parse.unquote(image_path)
@@ -70,7 +82,7 @@ async def upload_image(
     directory = default_images_directory(image_type)
     full_path = os.path.abspath(os.path.join(directory, image_dir, image_name))
     if os.path.commonprefix([directory, full_path]) != directory:
-        raise ControllerForbiddenError(f"Could not write image, '{image_path}' is forbidden")
+        raise ControllerForbiddenError(f"Cannot write image, '{image_path}' is forbidden")
 
     if await images_repo.get_image(image_path):
         raise ControllerBadRequestError(f"Image '{image_path}' already exists")
@@ -80,10 +92,24 @@ async def upload_image(
     except (OSError, InvalidImageError) as e:
         raise ControllerError(f"Could not save {image_type} image '{image_path}': {e}")
 
-    # TODO: automatically create template based on image checksum
-    #from gns3server.controller import Controller
-    #controller = Controller.instance()
-    #controller.appliance_manager.find_appliance_with_image(image.checksum)
+    try:
+        # attempt to automatically create a template based on image checksum
+        template = await Controller.instance().appliance_manager.install_appliance_from_image(
+            image.checksum,
+            images_repo,
+            directory
+        )
+
+        if template:
+            template_create = schemas.TemplateCreate(**template)
+            template = await TemplatesService(templates_repo).create_template(template_create)
+            template_id = template.get("template_id")
+            await rbac_repo.add_permission_to_user_with_path(current_user.user_id, f"/templates/{template_id}/*")
+            log.info(f"Template '{template.get('name')}' version {template.get('version')} "
+                     f"has been created using image '{image_name}'")
+
+    except (ControllerError, ValidationError, InvalidImageError) as e:
+        log.warning(f"Could not automatically create template using image '{image_path}': {e}")
 
     return image
 

gns3server/controller/appliance.py

@@ -68,6 +68,16 @@ class Appliance:
     def symbol(self, new_symbol):
         self._data["symbol"] = new_symbol
 
+    @property
+    def type(self):
+
+        if "iou" in self._data:
+            return "iou"
+        elif "dynamips" in self._data:
+            return "dynamips"
+        else:
+            return "qemu"
+
     def asdict(self):
         """
         Appliance data (a hash)

gns3server/controller/appliance_manager.py

@@ -16,10 +16,12 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import shutil
 import json
 import uuid
 import asyncio
+import aiofiles
+
+from aiohttp.client_exceptions import ClientError
 
 from .appliance import Appliance
 from ..config import Config
@@ -27,6 +29,9 @@ from ..utils.asyncio import locking
 from ..utils.get_resource import get_resource
 from ..utils.http_client import HTTPClient
 from .controller_error import ControllerError
+from .appliance_to_template import ApplianceToTemplate
+from ..utils.images import InvalidImageError, write_image, md5sum
+from ..utils.asyncio import wait_run_in_executor
 
 import logging
 
@@ -77,19 +82,89 @@ class ApplianceManager:
         os.makedirs(appliances_path, exist_ok=True)
         return appliances_path
 
-    #TODO: finish
-    def find_appliance_with_image(self, image_checksum):
+    def _find_appliance_from_image_checksum(self, image_checksum):
+        """
+        Find an appliance and version that matches an image checksum.
+        """
 
         for appliance in self._appliances.values():
             if appliance.images:
                 for image in appliance.images:
-                    if image["md5sum"] == image_checksum:
-                        print(f"APPLIANCE FOUND {appliance.name}")
-                        version = image["version"]
-                        print(f"IMAGE VERSION {version}")
-                        if image.versions:
-                            for version in image.versions:
-                                pass
+                    if image.get("md5sum") == image_checksum:
+                        return appliance, image.get("version")
+
+    async def _download_image(self, image_dir, image_name, image_type, image_url, images_repo):
+        """
+        Download an image.
+        """
+
+        log.info(f"Downloading image '{image_name}' from '{image_url}'")
+        image_path = os.path.join(image_dir, image_name)
+        try:
+            async with HTTPClient.get(image_url) as response:
+                if response.status != 200:
+                    raise ControllerError(f"Could not download '{image_name}' due to HTTP error code {response.status}")
+                await write_image(image_name, image_type, image_path, response.content.iter_any(), images_repo)
+        except (OSError, InvalidImageError) as e:
+            raise ControllerError(f"Could not save {image_type} image '{image_path}': {e}")
+        except ClientError as e:
+            raise ControllerError(f"Could not connect to download '{image_name}': {e}")
+        except asyncio.TimeoutError:
+            raise ControllerError(f"Timeout while downloading '{image_name}' from '{image_url}'")
+
+    async def _find_appliance_version_images(self, appliance, version, images_repo, image_dir):
+        """
+        Find all the images belonging to a specific appliance version.
+        """
+
+        version_images = version.get("images")
+        if version_images:
+            for appliance_key, appliance_file in version_images.items():
+                for image in appliance.images:
+                    if appliance_file == image.get("filename"):
+                        image_checksum = image.get("md5sum")
+                        image_in_db = await images_repo.get_image_by_checksum(image_checksum)
+                        if image_in_db:
+                            version_images[appliance_key] = image_in_db.filename
+                        else:
+                            # check if the image is on disk
+                            image_path = os.path.join(image_dir, appliance_file)
+                            if os.path.exists(image_path) and await wait_run_in_executor(md5sum, image_path) == image_checksum:
+                                async with aiofiles.open(image_path, "rb") as f:
+                                    await write_image(appliance_file, appliance.type, image_path, f, images_repo)
+                            else:
+                                # download the image if there is a direct download URL
+                                direct_download_url = image.get("direct_download_url")
+                                if direct_download_url:
+                                    await self._download_image(
+                                        image_dir,
+                                        appliance_file,
+                                        appliance.type,
+                                        direct_download_url,
+                                        images_repo)
+                                else:
+                                    raise ControllerError(f"Could not find '{appliance_file}'")
+
+    async def install_appliance_from_image(self, image_checksum, images_repo, image_dir):
+        """
+        Find the image checksum in appliance files
+        """
+
+        from . import Controller
+
+        appliance_info = self._find_appliance_from_image_checksum(image_checksum)
+        if appliance_info:
+            appliance, image_version = appliance_info
+            if appliance.versions:
+                for version in appliance.versions:
+                    if version.get("name") == image_version:
+                        await self._find_appliance_version_images(appliance, version, images_repo, image_dir)
+                        # downloading missing custom symbol for this appliance
+                        if appliance.symbol and not appliance.symbol.startswith(":/symbols/"):
+                            destination_path = os.path.join(Controller.instance().symbols.symbols_path(), appliance.symbol)
+                            if not os.path.exists(destination_path):
+                                await self._download_symbol(appliance.symbol, destination_path)
+                        return ApplianceToTemplate().new_template(appliance.asdict(), version, "local")  # FIXME: "local"
 
     def load_appliances(self, symbol_theme="Classic"):
         """
@@ -112,15 +187,17 @@ class ApplianceManager:
                     if not file.endswith(".gns3a") and not file.endswith(".gns3appliance"):
                         continue
                     path = os.path.join(directory, file)
-                    appliance_id = uuid.uuid3(
-                        uuid.NAMESPACE_URL, path
-                    )  # Generate UUID from path to avoid change between reboots
+                    # Generate UUID from path to avoid change between reboots
+                    appliance_id = uuid.uuid5(
+                        uuid.NAMESPACE_X500,
+                        path
+                    )
                     try:
                         with open(path, encoding="utf-8") as f:
                             appliance = Appliance(appliance_id, json.load(f), builtin=builtin)
                             json_data = appliance.asdict()  # Check if loaded without error
                             if appliance.status != "broken":
-                                self._appliances[appliance.id] = appliance
+                               self._appliances[appliance.id] = appliance
                             if not appliance.symbol or appliance.symbol.startswith(":/symbols/"):
                                 # apply a default symbol if the appliance has none or a default symbol
                                 default_symbol = self._get_default_symbol(json_data, symbol_theme)
@@ -171,6 +248,7 @@ class ApplianceManager:
         """
 
         symbol_url = f"https://raw.githubusercontent.com/GNS3/gns3-registry/master/symbols/{symbol}"
+        log.info(f"Downloading symbol '{symbol}'")
         async with HTTPClient.get(symbol_url) as response:
             if response.status != 200:
                 log.warning(

gns3server/controller/appliance_to_template.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2021 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class ApplianceToTemplate:
+    """
+    Appliance installation.
+    """
+
+    def new_template(self, appliance_config, version, server):
+        """
+        Creates a new template from an appliance.
+        """
+
+        new_template = {
+            "compute_id": server,
+            "name": appliance_config["name"],
+            "version": version.get("name")
+        }
+
+        if "usage" in appliance_config:
+            new_template["usage"] = appliance_config["usage"]
+
+        if appliance_config["category"] == "multilayer_switch":
+            new_template["category"] = "switch"
+        else:
+            new_template["category"] = appliance_config["category"]
+
+        if "symbol" in appliance_config:
+            new_template["symbol"] = appliance_config.get("symbol")
+
+        if new_template.get("symbol") is None:
+            if appliance_config["category"] == "guest":
+                if "docker" in appliance_config:
+                    new_template["symbol"] = ":/symbols/docker_guest.svg"
+                else:
+                    new_template["symbol"] = ":/symbols/qemu_guest.svg"
+            elif appliance_config["category"] == "router":
+                new_template["symbol"] = ":/symbols/router.svg"
+            elif appliance_config["category"] == "switch":
+                new_template["symbol"] = ":/symbols/ethernet_switch.svg"
+            elif appliance_config["category"] == "multilayer_switch":
+                new_template["symbol"] = ":/symbols/multilayer_switch.svg"
+            elif appliance_config["category"] == "firewall":
+                new_template["symbol"] = ":/symbols/firewall.svg"
+
+        if "qemu" in appliance_config:
+            new_template["template_type"] = "qemu"
+            self._add_qemu_config(new_template, appliance_config, version)
+        elif "iou" in appliance_config:
+            new_template["template_type"] = "iou"
+            self._add_iou_config(new_template, appliance_config, version)
+        elif "dynamips" in appliance_config:
+            new_template["template_type"] = "dynamips"
+            self._add_dynamips_config(new_template, appliance_config, version)
+        elif "docker" in appliance_config:
+            new_template["template_type"] = "docker"
+            self._add_docker_config(new_template, appliance_config)
+
+        return new_template
+
+    def _add_qemu_config(self, new_config, appliance_config, version):
+
+        new_config.update(appliance_config["qemu"])
+
+        # the following properties are not valid for a template
+        new_config.pop("kvm", None)
+        new_config.pop("path", None)
+        new_config.pop("arch", None)
+
+        options = appliance_config["qemu"].get("options", "")
+        if appliance_config["qemu"].get("kvm", "allow") == "disable" and "-machine accel=tcg" not in options:
+            options += " -machine accel=tcg"
+        new_config["options"] = options.strip()
+        new_config.update(version.get("images"))
+
+        if "path" in appliance_config["qemu"]:
+            new_config["qemu_path"] = appliance_config["qemu"]["path"]
+        else:
+            new_config["qemu_path"] = "qemu-system-{}".format(appliance_config["qemu"]["arch"])
+
+        if "first_port_name" in appliance_config:
+            new_config["first_port_name"] = appliance_config["first_port_name"]
+
+        if "port_name_format" in appliance_config:
+            new_config["port_name_format"] = appliance_config["port_name_format"]
+
+        if "port_segment_size" in appliance_config:
+            new_config["port_segment_size"] = appliance_config["port_segment_size"]
+
+        if "custom_adapters" in appliance_config:
+            new_config["custom_adapters"] = appliance_config["custom_adapters"]
+
+        if "linked_clone" in appliance_config:
+            new_config["linked_clone"] = appliance_config["linked_clone"]
+
+    def _add_docker_config(self, new_config, appliance_config):
+
+        new_config.update(appliance_config["docker"])
+
+        if "custom_adapters" in appliance_config:
+            new_config["custom_adapters"] = appliance_config["custom_adapters"]
+
+    def _add_dynamips_config(self, new_config, appliance_config, version):
+
+        new_config.update(appliance_config["dynamips"])
+        new_config["idlepc"] = version.get("idlepc", "")
+        new_config["image"] = version.get("images").get("image")
+
+    def _add_iou_config(self, new_config, appliance_config, version):
+
+        new_config.update(appliance_config["iou"])
+        new_config["path"] = version.get("images").get("image")

gns3server/controller/symbols.py

@@ -122,6 +122,10 @@ class Symbols:
                 return None
         return directory
 
+    def has_symbol(self, symbol_id):
+
+        return self._symbols_path.get(symbol_id)
+
     def get_path(self, symbol_id):
         try:
             return self._symbols_path[symbol_id]

gns3server/db/models/templates.py

@@ -29,6 +29,7 @@ class Template(BaseTable):
 
     template_id = Column(GUID, primary_key=True, default=generate_uuid)
     name = Column(String, index=True)
+    version = Column(String)
     category = Column(String)
     default_name_format = Column(String)
     symbol = Column(String)

gns3server/schemas/controller/templates/__init__.py

@@ -41,6 +41,7 @@ class TemplateBase(BaseModel):
 
     template_id: Optional[UUID] = None
     name: Optional[str] = None
+    version: Optional[str] = None
     category: Optional[Category] = None
     default_name_format: Optional[str] = None
     symbol: Optional[str] = None

gns3server/services/templates.py

@@ -58,7 +58,7 @@ DYNAMIPS_PLATFORM_TO_SHEMA = {
 # built-in templates have their compute_id set to None to tell clients to select a compute
 BUILTIN_TEMPLATES = [
     {
-        "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "cloud"),
+        "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "cloud"),
         "template_type": "cloud",
         "name": "Cloud",
         "default_name_format": "Cloud{0}",
@@ -68,7 +68,7 @@ BUILTIN_TEMPLATES = [
         "builtin": True,
     },
     {
-        "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "nat"),
+        "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "nat"),
         "template_type": "nat",
         "name": "NAT",
         "default_name_format": "NAT{0}",
@@ -78,7 +78,7 @@ BUILTIN_TEMPLATES = [
         "builtin": True,
     },
     {
-        "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "vpcs"),
+        "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "vpcs"),
         "template_type": "vpcs",
         "name": "VPCS",
         "default_name_format": "PC{0}",
@@ -89,7 +89,7 @@ BUILTIN_TEMPLATES = [
         "builtin": True,
     },
     {
-        "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "ethernet_switch"),
+        "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "ethernet_switch"),
         "template_type": "ethernet_switch",
         "name": "Ethernet switch",
         "console_type": "none",
@@ -100,7 +100,7 @@ BUILTIN_TEMPLATES = [
         "builtin": True,
     },
     {
-        "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "ethernet_hub"),
+        "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "ethernet_hub"),
         "template_type": "ethernet_hub",
         "name": "Ethernet hub",
         "default_name_format": "Hub{0}",
@@ -110,7 +110,7 @@ BUILTIN_TEMPLATES = [
         "builtin": True,
     },
     {
-        "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "frame_relay_switch"),
+        "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "frame_relay_switch"),
         "template_type": "frame_relay_switch",
         "name": "Frame Relay switch",
         "default_name_format": "FRSW{0}",
@@ -120,7 +120,7 @@ BUILTIN_TEMPLATES = [
         "builtin": True,
     },
     {
-        "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "atm_switch"),
+        "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "atm_switch"),
         "template_type": "atm_switch",
         "name": "ATM switch",
         "default_name_format": "ATMSW{0}",

gns3server/utils/images.py

@@ -237,8 +237,8 @@ def check_valid_image_header(data: bytes, image_type: str, header_magic_len: int
         if data[:header_magic_len] != b'\x7fELF\x01\x01\x01' and data[:7] != b'\x7fELF\x02\x01\x01':
             raise InvalidImageError("Invalid IOU file detected")
     elif image_type == "qemu":
-        if data[:header_magic_len] != b'QFI\xfb':
-            raise InvalidImageError("Invalid Qemu file detected (must be qcow2 format)")
+        if data[:header_magic_len] != b'QFI\xfb' and data[:header_magic_len] != b'KDMV':
+            raise InvalidImageError("Invalid Qemu file detected (must be qcow2 or VDMK format)")
 
 
 async def write_image(