From bc36d950605cd1be39bf0e1f81a536a86d4a7d0a Mon Sep 17 00:00:00 2001 From: grossmj Date: Sun, 10 Oct 2021 17:35:11 +1030 Subject: [PATCH] Finalize image management refactoring and auto install appliance if possible --- gns3server/api/routes/compute/qemu_nodes.py | 7 - gns3server/api/routes/controller/images.py | 38 ++++- gns3server/controller/appliance.py | 10 ++ gns3server/controller/appliance_manager.py | 106 ++++++++++++-- .../controller/appliance_to_template.py | 132 ++++++++++++++++++ gns3server/controller/symbols.py | 4 + gns3server/db/models/templates.py | 1 + .../schemas/controller/templates/__init__.py | 1 + gns3server/services/templates.py | 14 +- gns3server/utils/images.py | 4 +- 10 files changed, 281 insertions(+), 36 deletions(-) create mode 100644 gns3server/controller/appliance_to_template.py diff --git a/gns3server/api/routes/compute/qemu_nodes.py b/gns3server/api/routes/compute/qemu_nodes.py index 66ce046a..d60625d5 100644 --- a/gns3server/api/routes/compute/qemu_nodes.py +++ b/gns3server/api/routes/compute/qemu_nodes.py @@ -139,13 +139,6 @@ async def start_qemu_node(node: QemuVM = Depends(dep_node)) -> Response: Start a Qemu node. """ - qemu_manager = Qemu.instance() - hardware_accel = qemu_manager.config.settings.Qemu.enable_hardware_acceleration - if hardware_accel and "-machine accel=tcg" not in node.options: - pm = ProjectManager.instance() - if pm.check_hardware_virtualization(node) is False: - pass # FIXME: check this - # raise ComputeError("Cannot start VM with hardware acceleration (KVM/HAX) enabled because hardware virtualization (VT-x/AMD-V) is already used by another software like VMware or VirtualBox") await node.start() return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/gns3server/api/routes/controller/images.py b/gns3server/api/routes/controller/images.py index 6083e906..510b13db 100644 --- a/gns3server/api/routes/controller/images.py +++ b/gns3server/api/routes/controller/images.py @@ -26,9 +26,14 @@ from fastapi import APIRouter, Request, Response, Depends, status from sqlalchemy.orm.exc import MultipleResultsFound from typing import List from gns3server import schemas +from pydantic import ValidationError from gns3server.utils.images import InvalidImageError, default_images_directory, write_image from gns3server.db.repositories.images import ImagesRepository +from gns3server.db.repositories.templates import TemplatesRepository +from gns3server.services.templates import TemplatesService +from gns3server.db.repositories.rbac import RbacRepository +from gns3server.controller import Controller from gns3server.controller.controller_error import ( ControllerError, ControllerNotFoundError, @@ -36,6 +41,7 @@ from gns3server.controller.controller_error import ( ControllerBadRequestError ) +from .dependencies.authentication import get_current_active_user from .dependencies.database import get_repository log = logging.getLogger(__name__) @@ -43,7 +49,7 @@ log = logging.getLogger(__name__) router = APIRouter() -@router.get("") +@router.get("", response_model=List[schemas.Image]) async def get_images( images_repo: ImagesRepository = Depends(get_repository(ImagesRepository)), ) -> List[schemas.Image]: @@ -60,9 +66,15 @@ async def upload_image( request: Request, image_type: schemas.ImageType = schemas.ImageType.qemu, images_repo: ImagesRepository = Depends(get_repository(ImagesRepository)), + templates_repo: TemplatesRepository = Depends(get_repository(TemplatesRepository)), + current_user: schemas.User = Depends(get_current_active_user), + rbac_repo: RbacRepository = Depends(get_repository(RbacRepository)) ) -> schemas.Image: """ Upload an image. + + Example: curl -X POST http://host:port/v3/images/upload/my_image_name.qcow2?image_type=qemu \ + -H 'Authorization: Bearer ' --data-binary @"/path/to/image.qcow2" """ image_path = urllib.parse.unquote(image_path) @@ -70,7 +82,7 @@ async def upload_image( directory = default_images_directory(image_type) full_path = os.path.abspath(os.path.join(directory, image_dir, image_name)) if os.path.commonprefix([directory, full_path]) != directory: - raise ControllerForbiddenError(f"Could not write image, '{image_path}' is forbidden") + raise ControllerForbiddenError(f"Cannot write image, '{image_path}' is forbidden") if await images_repo.get_image(image_path): raise ControllerBadRequestError(f"Image '{image_path}' already exists") @@ -80,10 +92,24 @@ async def upload_image( except (OSError, InvalidImageError) as e: raise ControllerError(f"Could not save {image_type} image '{image_path}': {e}") - # TODO: automatically create template based on image checksum - #from gns3server.controller import Controller - #controller = Controller.instance() - #controller.appliance_manager.find_appliance_with_image(image.checksum) + try: + # attempt to automatically create a template based on image checksum + template = await Controller.instance().appliance_manager.install_appliance_from_image( + image.checksum, + images_repo, + directory + ) + + if template: + template_create = schemas.TemplateCreate(**template) + template = await TemplatesService(templates_repo).create_template(template_create) + template_id = template.get("template_id") + await rbac_repo.add_permission_to_user_with_path(current_user.user_id, f"/templates/{template_id}/*") + log.info(f"Template '{template.get('name')}' version {template.get('version')} " + f"has been created using image '{image_name}'") + + except (ControllerError, ValidationError, InvalidImageError) as e: + log.warning(f"Could not automatically create template using image '{image_path}': {e}") return image diff --git a/gns3server/controller/appliance.py b/gns3server/controller/appliance.py index 702e3ecf..c422f478 100644 --- a/gns3server/controller/appliance.py +++ b/gns3server/controller/appliance.py @@ -68,6 +68,16 @@ class Appliance: def symbol(self, new_symbol): self._data["symbol"] = new_symbol + @property + def type(self): + + if "iou" in self._data: + return "iou" + elif "dynamips" in self._data: + return "dynamips" + else: + return "qemu" + def asdict(self): """ Appliance data (a hash) diff --git a/gns3server/controller/appliance_manager.py b/gns3server/controller/appliance_manager.py index fb2645c8..a4f8f095 100644 --- a/gns3server/controller/appliance_manager.py +++ b/gns3server/controller/appliance_manager.py @@ -16,10 +16,12 @@ # along with this program. If not, see . import os -import shutil import json import uuid import asyncio +import aiofiles + +from aiohttp.client_exceptions import ClientError from .appliance import Appliance from ..config import Config @@ -27,6 +29,9 @@ from ..utils.asyncio import locking from ..utils.get_resource import get_resource from ..utils.http_client import HTTPClient from .controller_error import ControllerError +from .appliance_to_template import ApplianceToTemplate +from ..utils.images import InvalidImageError, write_image, md5sum +from ..utils.asyncio import wait_run_in_executor import logging @@ -77,19 +82,89 @@ class ApplianceManager: os.makedirs(appliances_path, exist_ok=True) return appliances_path - #TODO: finish - def find_appliance_with_image(self, image_checksum): + def _find_appliance_from_image_checksum(self, image_checksum): + """ + Find an appliance and version that matches an image checksum. + """ for appliance in self._appliances.values(): if appliance.images: for image in appliance.images: - if image["md5sum"] == image_checksum: - print(f"APPLIANCE FOUND {appliance.name}") - version = image["version"] - print(f"IMAGE VERSION {version}") - if image.versions: - for version in image.versions: - pass + if image.get("md5sum") == image_checksum: + return appliance, image.get("version") + + async def _download_image(self, image_dir, image_name, image_type, image_url, images_repo): + """ + Download an image. + """ + + log.info(f"Downloading image '{image_name}' from '{image_url}'") + image_path = os.path.join(image_dir, image_name) + try: + async with HTTPClient.get(image_url) as response: + if response.status != 200: + raise ControllerError(f"Could not download '{image_name}' due to HTTP error code {response.status}") + await write_image(image_name, image_type, image_path, response.content.iter_any(), images_repo) + except (OSError, InvalidImageError) as e: + raise ControllerError(f"Could not save {image_type} image '{image_path}': {e}") + except ClientError as e: + raise ControllerError(f"Could not connect to download '{image_name}': {e}") + except asyncio.TimeoutError: + raise ControllerError(f"Timeout while downloading '{image_name}' from '{image_url}'") + + async def _find_appliance_version_images(self, appliance, version, images_repo, image_dir): + """ + Find all the images belonging to a specific appliance version. + """ + + version_images = version.get("images") + if version_images: + for appliance_key, appliance_file in version_images.items(): + for image in appliance.images: + if appliance_file == image.get("filename"): + image_checksum = image.get("md5sum") + image_in_db = await images_repo.get_image_by_checksum(image_checksum) + if image_in_db: + version_images[appliance_key] = image_in_db.filename + else: + # check if the image is on disk + image_path = os.path.join(image_dir, appliance_file) + if os.path.exists(image_path) and await wait_run_in_executor(md5sum, image_path) == image_checksum: + async with aiofiles.open(image_path, "rb") as f: + await write_image(appliance_file, appliance.type, image_path, f, images_repo) + else: + # download the image if there is a direct download URL + direct_download_url = image.get("direct_download_url") + if direct_download_url: + await self._download_image( + image_dir, + appliance_file, + appliance.type, + direct_download_url, + images_repo) + else: + raise ControllerError(f"Could not find '{appliance_file}'") + + async def install_appliance_from_image(self, image_checksum, images_repo, image_dir): + """ + Find the image checksum in appliance files + """ + + from . import Controller + + appliance_info = self._find_appliance_from_image_checksum(image_checksum) + if appliance_info: + appliance, image_version = appliance_info + if appliance.versions: + for version in appliance.versions: + if version.get("name") == image_version: + await self._find_appliance_version_images(appliance, version, images_repo, image_dir) + # downloading missing custom symbol for this appliance + if appliance.symbol and not appliance.symbol.startswith(":/symbols/"): + destination_path = os.path.join(Controller.instance().symbols.symbols_path(), appliance.symbol) + if not os.path.exists(destination_path): + await self._download_symbol(appliance.symbol, destination_path) + return ApplianceToTemplate().new_template(appliance.asdict(), version, "local") # FIXME: "local" def load_appliances(self, symbol_theme="Classic"): """ @@ -112,15 +187,17 @@ class ApplianceManager: if not file.endswith(".gns3a") and not file.endswith(".gns3appliance"): continue path = os.path.join(directory, file) - appliance_id = uuid.uuid3( - uuid.NAMESPACE_URL, path - ) # Generate UUID from path to avoid change between reboots + # Generate UUID from path to avoid change between reboots + appliance_id = uuid.uuid5( + uuid.NAMESPACE_X500, + path + ) try: with open(path, encoding="utf-8") as f: appliance = Appliance(appliance_id, json.load(f), builtin=builtin) json_data = appliance.asdict() # Check if loaded without error if appliance.status != "broken": - self._appliances[appliance.id] = appliance + self._appliances[appliance.id] = appliance if not appliance.symbol or appliance.symbol.startswith(":/symbols/"): # apply a default symbol if the appliance has none or a default symbol default_symbol = self._get_default_symbol(json_data, symbol_theme) @@ -171,6 +248,7 @@ class ApplianceManager: """ symbol_url = f"https://raw.githubusercontent.com/GNS3/gns3-registry/master/symbols/{symbol}" + log.info(f"Downloading symbol '{symbol}'") async with HTTPClient.get(symbol_url) as response: if response.status != 200: log.warning( diff --git a/gns3server/controller/appliance_to_template.py b/gns3server/controller/appliance_to_template.py new file mode 100644 index 00000000..1cfc0c87 --- /dev/null +++ b/gns3server/controller/appliance_to_template.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# Copyright (C) 2021 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +import logging +log = logging.getLogger(__name__) + + +class ApplianceToTemplate: + """ + Appliance installation. + """ + + def new_template(self, appliance_config, version, server): + """ + Creates a new template from an appliance. + """ + + new_template = { + "compute_id": server, + "name": appliance_config["name"], + "version": version.get("name") + } + + if "usage" in appliance_config: + new_template["usage"] = appliance_config["usage"] + + if appliance_config["category"] == "multilayer_switch": + new_template["category"] = "switch" + else: + new_template["category"] = appliance_config["category"] + + if "symbol" in appliance_config: + new_template["symbol"] = appliance_config.get("symbol") + + if new_template.get("symbol") is None: + if appliance_config["category"] == "guest": + if "docker" in appliance_config: + new_template["symbol"] = ":/symbols/docker_guest.svg" + else: + new_template["symbol"] = ":/symbols/qemu_guest.svg" + elif appliance_config["category"] == "router": + new_template["symbol"] = ":/symbols/router.svg" + elif appliance_config["category"] == "switch": + new_template["symbol"] = ":/symbols/ethernet_switch.svg" + elif appliance_config["category"] == "multilayer_switch": + new_template["symbol"] = ":/symbols/multilayer_switch.svg" + elif appliance_config["category"] == "firewall": + new_template["symbol"] = ":/symbols/firewall.svg" + + if "qemu" in appliance_config: + new_template["template_type"] = "qemu" + self._add_qemu_config(new_template, appliance_config, version) + elif "iou" in appliance_config: + new_template["template_type"] = "iou" + self._add_iou_config(new_template, appliance_config, version) + elif "dynamips" in appliance_config: + new_template["template_type"] = "dynamips" + self._add_dynamips_config(new_template, appliance_config, version) + elif "docker" in appliance_config: + new_template["template_type"] = "docker" + self._add_docker_config(new_template, appliance_config) + + return new_template + + def _add_qemu_config(self, new_config, appliance_config, version): + + new_config.update(appliance_config["qemu"]) + + # the following properties are not valid for a template + new_config.pop("kvm", None) + new_config.pop("path", None) + new_config.pop("arch", None) + + options = appliance_config["qemu"].get("options", "") + if appliance_config["qemu"].get("kvm", "allow") == "disable" and "-machine accel=tcg" not in options: + options += " -machine accel=tcg" + new_config["options"] = options.strip() + new_config.update(version.get("images")) + + if "path" in appliance_config["qemu"]: + new_config["qemu_path"] = appliance_config["qemu"]["path"] + else: + new_config["qemu_path"] = "qemu-system-{}".format(appliance_config["qemu"]["arch"]) + + if "first_port_name" in appliance_config: + new_config["first_port_name"] = appliance_config["first_port_name"] + + if "port_name_format" in appliance_config: + new_config["port_name_format"] = appliance_config["port_name_format"] + + if "port_segment_size" in appliance_config: + new_config["port_segment_size"] = appliance_config["port_segment_size"] + + if "custom_adapters" in appliance_config: + new_config["custom_adapters"] = appliance_config["custom_adapters"] + + if "linked_clone" in appliance_config: + new_config["linked_clone"] = appliance_config["linked_clone"] + + def _add_docker_config(self, new_config, appliance_config): + + new_config.update(appliance_config["docker"]) + + if "custom_adapters" in appliance_config: + new_config["custom_adapters"] = appliance_config["custom_adapters"] + + def _add_dynamips_config(self, new_config, appliance_config, version): + + new_config.update(appliance_config["dynamips"]) + new_config["idlepc"] = version.get("idlepc", "") + new_config["image"] = version.get("images").get("image") + + def _add_iou_config(self, new_config, appliance_config, version): + + new_config.update(appliance_config["iou"]) + new_config["path"] = version.get("images").get("image") diff --git a/gns3server/controller/symbols.py b/gns3server/controller/symbols.py index 395fb947..4c727a26 100644 --- a/gns3server/controller/symbols.py +++ b/gns3server/controller/symbols.py @@ -122,6 +122,10 @@ class Symbols: return None return directory + def has_symbol(self, symbol_id): + + return self._symbols_path.get(symbol_id) + def get_path(self, symbol_id): try: return self._symbols_path[symbol_id] diff --git a/gns3server/db/models/templates.py b/gns3server/db/models/templates.py index 7b344735..3d5f939d 100644 --- a/gns3server/db/models/templates.py +++ b/gns3server/db/models/templates.py @@ -29,6 +29,7 @@ class Template(BaseTable): template_id = Column(GUID, primary_key=True, default=generate_uuid) name = Column(String, index=True) + version = Column(String) category = Column(String) default_name_format = Column(String) symbol = Column(String) diff --git a/gns3server/schemas/controller/templates/__init__.py b/gns3server/schemas/controller/templates/__init__.py index 74866b2e..14db9982 100644 --- a/gns3server/schemas/controller/templates/__init__.py +++ b/gns3server/schemas/controller/templates/__init__.py @@ -41,6 +41,7 @@ class TemplateBase(BaseModel): template_id: Optional[UUID] = None name: Optional[str] = None + version: Optional[str] = None category: Optional[Category] = None default_name_format: Optional[str] = None symbol: Optional[str] = None diff --git a/gns3server/services/templates.py b/gns3server/services/templates.py index aad23334..10535c71 100644 --- a/gns3server/services/templates.py +++ b/gns3server/services/templates.py @@ -58,7 +58,7 @@ DYNAMIPS_PLATFORM_TO_SHEMA = { # built-in templates have their compute_id set to None to tell clients to select a compute BUILTIN_TEMPLATES = [ { - "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "cloud"), + "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "cloud"), "template_type": "cloud", "name": "Cloud", "default_name_format": "Cloud{0}", @@ -68,7 +68,7 @@ BUILTIN_TEMPLATES = [ "builtin": True, }, { - "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "nat"), + "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "nat"), "template_type": "nat", "name": "NAT", "default_name_format": "NAT{0}", @@ -78,7 +78,7 @@ BUILTIN_TEMPLATES = [ "builtin": True, }, { - "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "vpcs"), + "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "vpcs"), "template_type": "vpcs", "name": "VPCS", "default_name_format": "PC{0}", @@ -89,7 +89,7 @@ BUILTIN_TEMPLATES = [ "builtin": True, }, { - "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "ethernet_switch"), + "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "ethernet_switch"), "template_type": "ethernet_switch", "name": "Ethernet switch", "console_type": "none", @@ -100,7 +100,7 @@ BUILTIN_TEMPLATES = [ "builtin": True, }, { - "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "ethernet_hub"), + "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "ethernet_hub"), "template_type": "ethernet_hub", "name": "Ethernet hub", "default_name_format": "Hub{0}", @@ -110,7 +110,7 @@ BUILTIN_TEMPLATES = [ "builtin": True, }, { - "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "frame_relay_switch"), + "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "frame_relay_switch"), "template_type": "frame_relay_switch", "name": "Frame Relay switch", "default_name_format": "FRSW{0}", @@ -120,7 +120,7 @@ BUILTIN_TEMPLATES = [ "builtin": True, }, { - "template_id": uuid.uuid3(uuid.NAMESPACE_DNS, "atm_switch"), + "template_id": uuid.uuid5(uuid.NAMESPACE_X500, "atm_switch"), "template_type": "atm_switch", "name": "ATM switch", "default_name_format": "ATMSW{0}", diff --git a/gns3server/utils/images.py b/gns3server/utils/images.py index 93ba9ef6..1c4c2e5b 100644 --- a/gns3server/utils/images.py +++ b/gns3server/utils/images.py @@ -237,8 +237,8 @@ def check_valid_image_header(data: bytes, image_type: str, header_magic_len: int if data[:header_magic_len] != b'\x7fELF\x01\x01\x01' and data[:7] != b'\x7fELF\x02\x01\x01': raise InvalidImageError("Invalid IOU file detected") elif image_type == "qemu": - if data[:header_magic_len] != b'QFI\xfb': - raise InvalidImageError("Invalid Qemu file detected (must be qcow2 format)") + if data[:header_magic_len] != b'QFI\xfb' and data[:header_magic_len] != b'KDMV': + raise InvalidImageError("Invalid Qemu file detected (must be qcow2 or VDMK format)") async def write_image(