From 04934691df9ef132af0d1396972c711685d44c7b Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 18 Oct 2021 18:04:30 +1030 Subject: [PATCH 1/6] Appliance management refactoring: * Install an appliance based on selected version * Each template have unique name and version * Allow to download an appliance file --- .../api/routes/controller/appliances.py | 63 ++++++- gns3server/api/routes/controller/images.py | 34 ++-- gns3server/controller/appliance.py | 16 +- gns3server/controller/appliance_manager.py | 165 ++++++++++++++---- .../controller/appliance_to_template.py | 4 +- gns3server/db/repositories/templates.py | 8 + gns3server/services/templates.py | 10 +- .../api/routes/controller/test_appliances.py | 48 ++++- tests/api/routes/controller/test_images.py | 26 +++ tests/api/routes/controller/test_templates.py | 28 ++- tests/resources/empty30G.qcow2 | Bin 0 -> 197120 bytes 11 files changed, 326 insertions(+), 76 deletions(-) create mode 100644 tests/resources/empty30G.qcow2 diff --git a/gns3server/api/routes/controller/appliances.py b/gns3server/api/routes/controller/appliances.py index 89dbe281..944b91c8 100644 --- a/gns3server/api/routes/controller/appliances.py +++ b/gns3server/api/routes/controller/appliances.py @@ -18,8 +18,25 @@ API routes for appliances. """ -from fastapi import APIRouter +import os +import logging + +from fastapi import APIRouter, Depends, Response, status +from fastapi.responses import FileResponse from typing import Optional, List +from uuid import UUID + +from gns3server import schemas +from gns3server.controller import Controller +from gns3server.controller.controller_error import ControllerNotFoundError +from gns3server.db.repositories.images import ImagesRepository +from gns3server.db.repositories.templates import TemplatesRepository +from gns3server.db.repositories.rbac import RbacRepository + +from .dependencies.authentication import get_current_active_user +from .dependencies.database import get_repository + +log = logging.getLogger(__name__) router = APIRouter() @@ -30,10 +47,50 @@ async def get_appliances(update: Optional[bool] = False, symbol_theme: Optional[ Return all appliances known by the controller. """ - from gns3server.controller import Controller - controller = Controller.instance() if update: await controller.appliance_manager.download_appliances() controller.appliance_manager.load_appliances(symbol_theme=symbol_theme) return [c.asdict() for c in controller.appliance_manager.appliances.values()] + + +@router.get("/{appliance_id}/download") +def download_appliance(appliance_id: UUID) -> FileResponse: + """ + Download an appliance file. + """ + + controller = Controller.instance() + appliance = controller.appliance_manager.appliances.get(str(appliance_id)) + if not appliance: + raise ControllerNotFoundError(message=f"Could not find appliance '{appliance_id}'") + + if not os.path.exists(appliance.path): + raise ControllerNotFoundError(message=f"Could not find appliance file '{appliance.path}'") + + return FileResponse(appliance.path, media_type="application/json") + + +@router.post("/{appliance_id}/install", status_code=status.HTTP_204_NO_CONTENT) +async def install_appliance( + appliance_id: UUID, + version: Optional[str] = None, + images_repo: ImagesRepository = Depends(get_repository(ImagesRepository)), + templates_repo: TemplatesRepository = Depends(get_repository(TemplatesRepository)), + current_user: schemas.User = Depends(get_current_active_user), + rbac_repo: RbacRepository = Depends(get_repository(RbacRepository)) +) -> Response: + """ + Install an appliance. + """ + + controller = Controller.instance() + await controller.appliance_manager.install_appliance( + appliance_id, + version, + images_repo, + templates_repo, + rbac_repo, + current_user + ) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/gns3server/api/routes/controller/images.py b/gns3server/api/routes/controller/images.py index 510b13db..d8d8e193 100644 --- a/gns3server/api/routes/controller/images.py +++ b/gns3server/api/routes/controller/images.py @@ -24,14 +24,12 @@ import urllib.parse from fastapi import APIRouter, Request, Response, Depends, status from sqlalchemy.orm.exc import MultipleResultsFound -from typing import List +from typing import List, Optional from gns3server import schemas -from pydantic import ValidationError from gns3server.utils.images import InvalidImageError, default_images_directory, write_image from gns3server.db.repositories.images import ImagesRepository from gns3server.db.repositories.templates import TemplatesRepository -from gns3server.services.templates import TemplatesService from gns3server.db.repositories.rbac import RbacRepository from gns3server.controller import Controller from gns3server.controller.controller_error import ( @@ -68,7 +66,8 @@ async def upload_image( images_repo: ImagesRepository = Depends(get_repository(ImagesRepository)), templates_repo: TemplatesRepository = Depends(get_repository(TemplatesRepository)), current_user: schemas.User = Depends(get_current_active_user), - rbac_repo: RbacRepository = Depends(get_repository(RbacRepository)) + rbac_repo: RbacRepository = Depends(get_repository(RbacRepository)), + install_appliances: Optional[bool] = True ) -> schemas.Image: """ Upload an image. @@ -92,25 +91,18 @@ async def upload_image( except (OSError, InvalidImageError) as e: raise ControllerError(f"Could not save {image_type} image '{image_path}': {e}") - try: - # attempt to automatically create a template based on image checksum - template = await Controller.instance().appliance_manager.install_appliance_from_image( + if install_appliances: + # attempt to automatically create templates based on image checksum + await Controller.instance().appliance_manager.install_appliances_from_image( + image_path, image.checksum, images_repo, + templates_repo, + rbac_repo, + current_user, directory ) - if template: - template_create = schemas.TemplateCreate(**template) - template = await TemplatesService(templates_repo).create_template(template_create) - template_id = template.get("template_id") - await rbac_repo.add_permission_to_user_with_path(current_user.user_id, f"/templates/{template_id}/*") - log.info(f"Template '{template.get('name')}' version {template.get('version')} " - f"has been created using image '{image_name}'") - - except (ControllerError, ValidationError, InvalidImageError) as e: - log.warning(f"Could not automatically create template using image '{image_path}': {e}") - return image @@ -150,8 +142,10 @@ async def delete_image( if not image: raise ControllerNotFoundError(f"Image '{image_path}' not found") - if await images_repo.get_image_templates(image.image_id): - raise ControllerError(f"Image '{image_path}' is used by one or more templates") + templates = await images_repo.get_image_templates(image.image_id) + if templates: + template_names = ", ".join([template.name for template in templates]) + raise ControllerError(f"Image '{image_path}' is used by one or more templates: {template_names}") try: os.remove(image.path) diff --git a/gns3server/controller/appliance.py b/gns3server/controller/appliance.py index c422f478..c460e4e9 100644 --- a/gns3server/controller/appliance.py +++ b/gns3server/controller/appliance.py @@ -24,15 +24,11 @@ log = logging.getLogger(__name__) class Appliance: - def __init__(self, appliance_id, data, builtin=True): + def __init__(self, path, data, builtin=True): - if appliance_id is None: - self._id = str(uuid.uuid4()) - elif isinstance(appliance_id, uuid.UUID): - self._id = str(appliance_id) - else: - self._id = appliance_id self._data = data.copy() + self._id = data.get("appliance_id", uuid.uuid5(uuid.NAMESPACE_X500, path)) + self._path = path self._builtin = builtin if "appliance_id" in self._data: del self._data["appliance_id"] @@ -44,6 +40,10 @@ class Appliance: def id(self): return self._id + @property + def path(self): + return self._path + @property def status(self): return self._data["status"] @@ -75,6 +75,8 @@ class Appliance: return "iou" elif "dynamips" in self._data: return "dynamips" + elif "docker" in self._data: + return "docker" else: return "qemu" diff --git a/gns3server/controller/appliance_manager.py b/gns3server/controller/appliance_manager.py index a4f8f095..d837abe2 100644 --- a/gns3server/controller/appliance_manager.py +++ b/gns3server/controller/appliance_manager.py @@ -17,22 +17,32 @@ import os import json -import uuid import asyncio import aiofiles +from typing import Tuple, List from aiohttp.client_exceptions import ClientError +from uuid import UUID +from pydantic import ValidationError + from .appliance import Appliance from ..config import Config from ..utils.asyncio import locking from ..utils.get_resource import get_resource from ..utils.http_client import HTTPClient -from .controller_error import ControllerError +from .controller_error import ControllerBadRequestError, ControllerNotFoundError, ControllerError from .appliance_to_template import ApplianceToTemplate from ..utils.images import InvalidImageError, write_image, md5sum from ..utils.asyncio import wait_run_in_executor +from gns3server import schemas +from gns3server.utils.images import default_images_directory +from gns3server.db.repositories.images import ImagesRepository +from gns3server.db.repositories.templates import TemplatesRepository +from gns3server.services.templates import TemplatesService +from gns3server.db.repositories.rbac import RbacRepository + import logging log = logging.getLogger(__name__) @@ -49,7 +59,7 @@ class ApplianceManager: self._appliances_etag = None @property - def appliances_etag(self): + def appliances_etag(self) -> str: """ :returns: ETag for downloaded appliances """ @@ -65,14 +75,14 @@ class ApplianceManager: self._appliances_etag = etag @property - def appliances(self): + def appliances(self) -> dict: """ :returns: The dictionary of appliances managed by GNS3 """ return self._appliances - def appliances_path(self): + def appliances_path(self) -> str: """ Get the image storage directory """ @@ -82,18 +92,27 @@ class ApplianceManager: os.makedirs(appliances_path, exist_ok=True) return appliances_path - def _find_appliance_from_image_checksum(self, image_checksum): + def _find_appliances_from_image_checksum(self, image_checksum: str) -> List[Tuple[Appliance, str]]: """ - Find an appliance and version that matches an image checksum. + Find appliances that matches an image checksum. """ + appliances = [] for appliance in self._appliances.values(): if appliance.images: for image in appliance.images: if image.get("md5sum") == image_checksum: - return appliance, image.get("version") + appliances.append((appliance, image.get("version"))) + return appliances - async def _download_image(self, image_dir, image_name, image_type, image_url, images_repo): + async def _download_image( + self, + image_dir: str, + image_name: str, + image_type: str, + image_url: str, + images_repo: ImagesRepository + ) -> None: """ Download an image. """ @@ -112,7 +131,13 @@ class ApplianceManager: except asyncio.TimeoutError: raise ControllerError(f"Timeout while downloading '{image_name}' from '{image_url}'") - async def _find_appliance_version_images(self, appliance, version, images_repo, image_dir): + async def _find_appliance_version_images( + self, + appliance: Appliance, + version: dict, + images_repo: ImagesRepository, + image_dir: str + ) -> None: """ Find all the images belonging to a specific appliance version. """ @@ -145,28 +170,103 @@ class ApplianceManager: else: raise ControllerError(f"Could not find '{appliance_file}'") - async def install_appliance_from_image(self, image_checksum, images_repo, image_dir): + async def _create_template(self, template_data, templates_repo, rbac_repo, current_user): """ - Find the image checksum in appliance files + Create a new template + """ + + try: + template_create = schemas.TemplateCreate(**template_data) + except ValidationError as e: + raise ControllerError(message=f"Could not validate template data: {e}") + template = await TemplatesService(templates_repo).create_template(template_create) + template_id = template.get("template_id") + await rbac_repo.add_permission_to_user_with_path(current_user.user_id, f"/templates/{template_id}/*") + log.info(f"Template '{template.get('name')}' has been created") + + async def _appliance_to_template(self, appliance: Appliance, version: str = None) -> dict: + """ + Get template data from appliance """ from . import Controller - appliance_info = self._find_appliance_from_image_checksum(image_checksum) - if appliance_info: - appliance, image_version = appliance_info + # downloading missing custom symbol for this appliance + if appliance.symbol and not appliance.symbol.startswith(":/symbols/"): + destination_path = os.path.join(Controller.instance().symbols.symbols_path(), appliance.symbol) + if not os.path.exists(destination_path): + await self._download_symbol(appliance.symbol, destination_path) + return ApplianceToTemplate().new_template(appliance.asdict(), version, "local") # FIXME: "local" + + async def install_appliances_from_image( + self, + image_path: str, + image_checksum: str, + images_repo: ImagesRepository, + templates_repo: TemplatesRepository, + rbac_repo: RbacRepository, + current_user: schemas.User, + image_dir: str + ) -> None: + """ + Install appliances using an image checksum + """ + + appliances_info = self._find_appliances_from_image_checksum(image_checksum) + for appliance, image_version in appliances_info: if appliance.versions: for version in appliance.versions: if version.get("name") == image_version: - await self._find_appliance_version_images(appliance, version, images_repo, image_dir) - # downloading missing custom symbol for this appliance - if appliance.symbol and not appliance.symbol.startswith(":/symbols/"): - destination_path = os.path.join(Controller.instance().symbols.symbols_path(), appliance.symbol) - if not os.path.exists(destination_path): - await self._download_symbol(appliance.symbol, destination_path) - return ApplianceToTemplate().new_template(appliance.asdict(), version, "local") # FIXME: "local" + try: + await self._find_appliance_version_images(appliance, version, images_repo, image_dir) + template_data = await self._appliance_to_template(appliance, version) + await self._create_template(template_data, templates_repo, rbac_repo, current_user) + except (ControllerError, InvalidImageError) as e: + log.warning(f"Could not automatically create template using image '{image_path}': {e}") - def load_appliances(self, symbol_theme="Classic"): + async def install_appliance( + self, + appliance_id: UUID, + version: str, + images_repo: ImagesRepository, + templates_repo: TemplatesRepository, + rbac_repo: RbacRepository, + current_user: schemas.User + ) -> None: + """ + Install a new appliance + """ + + appliance = self._appliances.get(str(appliance_id)) + if not appliance: + raise ControllerNotFoundError(message=f"Could not find appliance '{appliance_id}'") + + if version: + if not appliance.versions: + raise ControllerBadRequestError(message=f"Appliance '{appliance_id}' do not have versions") + + image_dir = default_images_directory(appliance.type) + for appliance_version_info in appliance.versions: + if appliance_version_info.get("name") == version: + try: + await self._find_appliance_version_images(appliance, appliance_version_info, images_repo, image_dir) + except InvalidImageError as e: + raise ControllerError(message=f"Image error: {e}") + template_data = await self._appliance_to_template(appliance, appliance_version_info) + return await self._create_template(template_data, templates_repo, rbac_repo, current_user) + + raise ControllerNotFoundError(message=f"Could not find version '{version}' in appliance '{appliance_id}'") + + else: + if appliance.versions: + # TODO: install appliance versions based on available images + raise ControllerBadRequestError(message=f"Selecting a version is required to install " + f"appliance '{appliance_id}'") + + template_data = await self._appliance_to_template(appliance) + await self._create_template(template_data, templates_repo, rbac_repo, current_user) + + def load_appliances(self, symbol_theme: str = "Classic") -> None: """ Loads appliance files from disk. """ @@ -187,27 +287,22 @@ class ApplianceManager: if not file.endswith(".gns3a") and not file.endswith(".gns3appliance"): continue path = os.path.join(directory, file) - # Generate UUID from path to avoid change between reboots - appliance_id = uuid.uuid5( - uuid.NAMESPACE_X500, - path - ) try: with open(path, encoding="utf-8") as f: - appliance = Appliance(appliance_id, json.load(f), builtin=builtin) + appliance = Appliance(path, json.load(f), builtin=builtin) json_data = appliance.asdict() # Check if loaded without error if appliance.status != "broken": - self._appliances[appliance.id] = appliance + self._appliances[appliance.id] = appliance if not appliance.symbol or appliance.symbol.startswith(":/symbols/"): # apply a default symbol if the appliance has none or a default symbol default_symbol = self._get_default_symbol(json_data, symbol_theme) if default_symbol: appliance.symbol = default_symbol except (ValueError, OSError, KeyError) as e: - log.warning("Cannot load appliance file '%s': %s", path, str(e)) + log.warning(f"Cannot load appliance file '{path}': {e}") continue - def _get_default_symbol(self, appliance, symbol_theme): + def _get_default_symbol(self, appliance: dict, symbol_theme: str) -> str: """ Returns the default symbol for a given appliance. """ @@ -223,7 +318,7 @@ class ApplianceManager: return controller.symbols.get_default_symbol("qemu_guest", symbol_theme) return controller.symbols.get_default_symbol(category, symbol_theme) - async def download_custom_symbols(self): + async def download_custom_symbols(self) -> None: """ Download custom appliance symbols from our GitHub registry repository. """ @@ -242,7 +337,7 @@ class ApplianceManager: # refresh the symbol cache Controller.instance().symbols.list() - async def _download_symbol(self, symbol, destination_path): + async def _download_symbol(self, symbol: str, destination_path: str) -> None: """ Download a custom appliance symbol from our GitHub registry repository. """ @@ -266,7 +361,7 @@ class ApplianceManager: log.warning(f"Could not write appliance symbol '{destination_path}': {e}") @locking - async def download_appliances(self): + async def download_appliances(self) -> None: """ Downloads appliance files from GitHub registry repository. """ diff --git a/gns3server/controller/appliance_to_template.py b/gns3server/controller/appliance_to_template.py index 1cfc0c87..aee787f6 100644 --- a/gns3server/controller/appliance_to_template.py +++ b/gns3server/controller/appliance_to_template.py @@ -34,9 +34,11 @@ class ApplianceToTemplate: new_template = { "compute_id": server, "name": appliance_config["name"], - "version": version.get("name") } + if version: + new_template["version"] = version.get("name") + if "usage" in appliance_config: new_template["usage"] = appliance_config["usage"] diff --git a/gns3server/db/repositories/templates.py b/gns3server/db/repositories/templates.py index f24608ca..8fb3e4cf 100644 --- a/gns3server/db/repositories/templates.py +++ b/gns3server/db/repositories/templates.py @@ -57,6 +57,14 @@ class TemplatesRepository(BaseRepository): result = await self._db_session.execute(query) return result.scalars().first() + async def get_template_by_name_and_version(self, name: str, version: str) -> Union[None, models.Template]: + + query = select(models.Template).\ + options(selectinload(models.Template.images)).\ + where(models.Template.name == name, models.Template.version == version) + result = await self._db_session.execute(query) + return result.scalars().first() + async def get_templates(self) -> List[models.Template]: query = select(models.Template).options(selectinload(models.Template.images)) diff --git a/gns3server/services/templates.py b/gns3server/services/templates.py index 10535c71..569f70e8 100644 --- a/gns3server/services/templates.py +++ b/gns3server/services/templates.py @@ -25,8 +25,8 @@ from typing import List from gns3server import schemas import gns3server.db.models as models from gns3server.db.repositories.templates import TemplatesRepository -from gns3server.controller import Controller from gns3server.controller.controller_error import ( + ControllerError, ControllerBadRequestError, ControllerNotFoundError, ControllerForbiddenError, @@ -137,6 +137,7 @@ class TemplatesService: def __init__(self, templates_repo: TemplatesRepository): self._templates_repo = templates_repo + from gns3server.controller import Controller self._controller = Controller.instance() def get_builtin_template(self, template_id: UUID) -> dict: @@ -195,6 +196,13 @@ class TemplatesService: async def create_template(self, template_create: schemas.TemplateCreate) -> dict: + if await self._templates_repo.get_template_by_name_and_version(template_create.name, template_create.version): + if template_create.version: + raise ControllerError(f"A template with name '{template_create.name}' and " + f"version {template_create.version} already exists") + else: + raise ControllerError(f"A template with name '{template_create.name}' already exists") + try: # get the default template settings template_settings = jsonable_encoder(template_create, exclude_unset=True) diff --git a/tests/api/routes/controller/test_appliances.py b/tests/api/routes/controller/test_appliances.py index 04c429d3..53fcc73b 100644 --- a/tests/api/routes/controller/test_appliances.py +++ b/tests/api/routes/controller/test_appliances.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (C) 2020 GNS3 Technologies Inc. +# Copyright (C) 2021 GNS3 Technologies Inc. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,7 +15,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os import pytest +import shutil from fastapi import FastAPI, status from httpx import AsyncClient @@ -23,8 +25,44 @@ from httpx import AsyncClient pytestmark = pytest.mark.asyncio -async def test_appliances_list(app: FastAPI, client: AsyncClient) -> None: +class TestApplianceRoutes: - response = await client.get(app.url_path_for("get_appliances")) - assert response.status_code == status.HTTP_200_OK - assert len(response.json()) > 0 + async def test_appliances_list(self, app: FastAPI, client: AsyncClient) -> None: + + response = await client.get(app.url_path_for("get_appliances")) + assert response.status_code == status.HTTP_200_OK + assert len(response.json()) > 0 + + async def test_appliance_download(self, app: FastAPI, client: AsyncClient) -> None: + + appliance_id = "3bf492b6-5717-4257-9bfd-b34617c6f133" # Cisco IOSv appliance + response = await client.get(app.url_path_for("download_appliance", appliance_id=appliance_id)) + assert response.status_code == status.HTTP_200_OK + assert response.json()["appliance_id"] == appliance_id + + async def test_docker_appliance_install(self, app: FastAPI, client: AsyncClient) -> None: + + appliance_id = "fc520ae2-a4e5-48c3-9a13-516bb2e94668" # Alpine Linux appliance + response = await client.post(app.url_path_for("install_appliance", appliance_id=appliance_id)) + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_docker_appliance_install_with_version(self, app: FastAPI, client: AsyncClient) -> None: + + appliance_id = "fc520ae2-a4e5-48c3-9a13-516bb2e94668" # Alpine Linux appliance + params = {"version": "123"} + response = await client.post(app.url_path_for("install_appliance", appliance_id=appliance_id), params=params) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + async def test_qemu_appliance_install_with_version(self, app: FastAPI, client: AsyncClient, images_dir: str) -> None: + + shutil.copy("tests/resources/empty8G.qcow2", os.path.join(images_dir, "QEMU", "empty8G.qcow2")) + appliance_id = "1cfdf900-7c30-4cb7-8f03-3f61d2581633" # Empty VM appliance + params = {"version": "8G"} + response = await client.post(app.url_path_for("install_appliance", appliance_id=appliance_id), params=params) + assert response.status_code == status.HTTP_204_NO_CONTENT + + async def test_qemu_appliance_install_without_version(self, app: FastAPI, client: AsyncClient, images_dir: str) -> None: + + appliance_id = "1cfdf900-7c30-4cb7-8f03-3f61d2581633" # Empty VM appliance + response = await client.post(app.url_path_for("install_appliance", appliance_id=appliance_id)) + assert response.status_code == status.HTTP_400_BAD_REQUEST diff --git a/tests/api/routes/controller/test_images.py b/tests/api/routes/controller/test_images.py index 2a4a2230..1423957b 100644 --- a/tests/api/routes/controller/test_images.py +++ b/tests/api/routes/controller/test_images.py @@ -23,7 +23,9 @@ from sqlalchemy.ext.asyncio import AsyncSession from fastapi import FastAPI, status from httpx import AsyncClient +from gns3server.controller import Controller from gns3server.db.repositories.images import ImagesRepository +from gns3server.db.repositories.templates import TemplatesRepository pytestmark = pytest.mark.asyncio @@ -256,3 +258,27 @@ class TestImageRoutes: images_repo = ImagesRepository(db_session) images_in_db = await images_repo.get_images() assert len(images_in_db) == 0 + + async def test_image_upload_create_appliance( + self, app: FastAPI, + client: AsyncClient, + db_session: AsyncSession, + controller: Controller + ) -> None: + + controller.appliance_manager.load_appliances() # make sure appliances are loaded + image_path = "tests/resources/empty30G.qcow2" + image_name = os.path.basename(image_path) + with open(image_path, "rb") as f: + image_data = f.read() + response = await client.post( + app.url_path_for("upload_image", image_path=image_name), + params={"image_type": "qemu"}, + content=image_data) + assert response.status_code == status.HTTP_201_CREATED + + templates_repo = TemplatesRepository(db_session) + templates = await templates_repo.get_templates() + assert len(templates) == 1 + assert templates[0].name == "Empty VM" + assert templates[0].version == "30G" diff --git a/tests/api/routes/controller/test_templates.py b/tests/api/routes/controller/test_templates.py index d2423230..2d357959 100644 --- a/tests/api/routes/controller/test_templates.py +++ b/tests/api/routes/controller/test_templates.py @@ -62,6 +62,7 @@ class TestTemplateRoutes: template_id = str(uuid.uuid4()) params = {"template_id": template_id, "name": "VPCS_TEST", + "version": "1.0", "compute_id": "local", "template_type": "vpcs"} @@ -72,9 +73,25 @@ class TestTemplateRoutes: assert response.status_code == status.HTTP_200_OK assert response.json()["template_id"] == template_id + async def test_template_create_same_name_and_version( + self, + app: FastAPI, + client: AsyncClient, + controller: Controller + ) -> None: + + params = {"name": "VPCS_TEST", + "version": "1.0", + "compute_id": "local", + "template_type": "vpcs"} + + response = await client.post(app.url_path_for("create_template"), json=params) + assert response.status_code == status.HTTP_409_CONFLICT + async def test_template_create_wrong_type(self, app: FastAPI, client: AsyncClient, controller: Controller) -> None: params = {"name": "VPCS_TEST", + "version": "2.0", "compute_id": "local", "template_type": "invalid_template_type"} @@ -86,6 +103,7 @@ class TestTemplateRoutes: template_id = str(uuid.uuid4()) params = {"template_id": template_id, "name": "VPCS_TEST", + "version": "3.0", "compute_id": "local", "template_type": "vpcs"} @@ -107,6 +125,7 @@ class TestTemplateRoutes: template_id = str(uuid.uuid4()) params = {"template_id": template_id, "name": "VPCS_TEST", + "version": "4.0", "compute_id": "local", "template_type": "vpcs"} @@ -426,7 +445,7 @@ class TestDynamipsTemplate: async def test_c3600_dynamips_template_create_wrong_chassis(self, app: FastAPI, client: AsyncClient) -> None: - params = {"name": "Cisco c3600 template", + params = {"name": "Cisco c3600 template with wrong chassis", "platform": "c3600", "chassis": "3650", "compute_id": "local", @@ -530,7 +549,7 @@ class TestDynamipsTemplate: async def test_c2600_dynamips_template_create_wrong_chassis(self, app: FastAPI, client: AsyncClient) -> None: - params = {"name": "Cisco c2600 template", + params = {"name": "Cisco c2600 template with wrong chassis", "platform": "c2600", "chassis": "2660XM", "compute_id": "local", @@ -589,7 +608,7 @@ class TestDynamipsTemplate: async def test_c1700_dynamips_template_create_wrong_chassis(self, app: FastAPI, client: AsyncClient) -> None: - params = {"name": "Cisco c1700 template", + params = {"name": "Cisco c1700 template with wrong chassis", "platform": "c1700", "chassis": "1770", "compute_id": "local", @@ -1200,6 +1219,7 @@ class TestImageAssociationWithTemplate: ) -> None: params = {"name": "Qemu template", + "version": "1.0", "compute_id": "local", "platform": "i386", "hda_disk_image": "subdir/image.qcow2", @@ -1224,7 +1244,7 @@ class TestImageAssociationWithTemplate: async def test_template_create_with_non_existing_image(self, app: FastAPI, client: AsyncClient) -> None: - params = {"name": "Qemu template", + params = {"name": "Qemu template with non existing image", "compute_id": "local", "platform": "i386", "hda_disk_image": "unkown_image.qcow2", diff --git a/tests/resources/empty30G.qcow2 b/tests/resources/empty30G.qcow2 new file mode 100644 index 0000000000000000000000000000000000000000..b5ff3f5621518e413f4e04a866fb1b9d1fb0eff2 GIT binary patch literal 197120 zcmeIuu?m175CBlEptTQb4jTK5KBNC`NtCd~Ifthxciw@=JGT7}A&lvK*OHR?sxNB} z->#Uma@U?#G^N7XDtaeCfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+0D+DKhK^=23IPHH2oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF{38%U+5i0RR{{hG5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV8o4 Ffd`gV11SIi literal 0 HcmV?d00001 From 13ff7df9fa196718139a50a1c7cea458c1192279 Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 18 Oct 2021 21:46:50 +1030 Subject: [PATCH 2/6] Validate appliance files with Pydantic --- .../api/routes/controller/appliances.py | 19 +- gns3server/controller/appliance.py | 7 +- gns3server/controller/appliance_manager.py | 19 +- gns3server/schemas/__init__.py | 1 + gns3server/schemas/controller/appliances.py | 463 ++++++++++++++++++ .../api/routes/controller/test_appliances.py | 4 +- 6 files changed, 491 insertions(+), 22 deletions(-) create mode 100644 gns3server/schemas/controller/appliances.py diff --git a/gns3server/api/routes/controller/appliances.py b/gns3server/api/routes/controller/appliances.py index 944b91c8..18532e05 100644 --- a/gns3server/api/routes/controller/appliances.py +++ b/gns3server/api/routes/controller/appliances.py @@ -18,11 +18,9 @@ API routes for appliances. """ -import os import logging from fastapi import APIRouter, Depends, Response, status -from fastapi.responses import FileResponse from typing import Optional, List from uuid import UUID @@ -42,7 +40,10 @@ router = APIRouter() @router.get("") -async def get_appliances(update: Optional[bool] = False, symbol_theme: Optional[str] = "Classic") -> List[dict]: +async def get_appliances( + update: Optional[bool] = False, + symbol_theme: Optional[str] = "Classic" +) -> List[schemas.Appliance]: """ Return all appliances known by the controller. """ @@ -54,21 +55,17 @@ async def get_appliances(update: Optional[bool] = False, symbol_theme: Optional[ return [c.asdict() for c in controller.appliance_manager.appliances.values()] -@router.get("/{appliance_id}/download") -def download_appliance(appliance_id: UUID) -> FileResponse: +@router.get("/{appliance_id}") +def get_appliance(appliance_id: UUID) -> schemas.Appliance: """ - Download an appliance file. + Get an appliance file. """ controller = Controller.instance() appliance = controller.appliance_manager.appliances.get(str(appliance_id)) if not appliance: raise ControllerNotFoundError(message=f"Could not find appliance '{appliance_id}'") - - if not os.path.exists(appliance.path): - raise ControllerNotFoundError(message=f"Could not find appliance file '{appliance.path}'") - - return FileResponse(appliance.path, media_type="application/json") + return appliance.asdict() @router.post("/{appliance_id}/install", status_code=status.HTTP_204_NO_CONTENT) diff --git a/gns3server/controller/appliance.py b/gns3server/controller/appliance.py index c460e4e9..9e93ee42 100644 --- a/gns3server/controller/appliance.py +++ b/gns3server/controller/appliance.py @@ -16,7 +16,6 @@ # along with this program. If not, see . import copy -import uuid import logging log = logging.getLogger(__name__) @@ -27,12 +26,9 @@ class Appliance: def __init__(self, path, data, builtin=True): self._data = data.copy() - self._id = data.get("appliance_id", uuid.uuid5(uuid.NAMESPACE_X500, path)) + self._id = self._data.get("appliance_id") self._path = path self._builtin = builtin - if "appliance_id" in self._data: - del self._data["appliance_id"] - if self.status != "broken": log.debug(f'Appliance "{self.name}" [{self._id}] loaded') @@ -84,6 +80,7 @@ class Appliance: """ Appliance data (a hash) """ + data = copy.deepcopy(self._data) data["builtin"] = self._builtin return data diff --git a/gns3server/controller/appliance_manager.py b/gns3server/controller/appliance_manager.py index d837abe2..c00b26fd 100644 --- a/gns3server/controller/appliance_manager.py +++ b/gns3server/controller/appliance_manager.py @@ -214,6 +214,10 @@ class ApplianceManager: appliances_info = self._find_appliances_from_image_checksum(image_checksum) for appliance, image_version in appliances_info: + try: + schemas.Appliance.parse_obj(appliance.asdict()) + except ValidationError as e: + log.warning(message=f"Could not validate appliance '{appliance.id}': {e}") if appliance.versions: for version in appliance.versions: if version.get("name") == image_version: @@ -241,6 +245,11 @@ class ApplianceManager: if not appliance: raise ControllerNotFoundError(message=f"Could not find appliance '{appliance_id}'") + try: + schemas.Appliance.parse_obj(appliance.asdict()) + except ValidationError as e: + raise ControllerError(message=f"Could not validate appliance '{appliance_id}': {e}") + if version: if not appliance.versions: raise ControllerBadRequestError(message=f"Appliance '{appliance_id}' do not have versions") @@ -289,16 +298,18 @@ class ApplianceManager: path = os.path.join(directory, file) try: with open(path, encoding="utf-8") as f: - appliance = Appliance(path, json.load(f), builtin=builtin) - json_data = appliance.asdict() # Check if loaded without error + json_data = json.load(f) + schemas.Appliance.parse_obj(json_data) + appliance = Appliance(path, json_data, builtin=builtin) + appliance_data = appliance.asdict() # Check if loaded without error if appliance.status != "broken": self._appliances[appliance.id] = appliance if not appliance.symbol or appliance.symbol.startswith(":/symbols/"): # apply a default symbol if the appliance has none or a default symbol - default_symbol = self._get_default_symbol(json_data, symbol_theme) + default_symbol = self._get_default_symbol(appliance_data, symbol_theme) if default_symbol: appliance.symbol = default_symbol - except (ValueError, OSError, KeyError) as e: + except (ValueError, OSError, KeyError, ValidationError) as e: log.warning(f"Cannot load appliance file '{path}': {e}") continue diff --git a/gns3server/schemas/__init__.py b/gns3server/schemas/__init__.py index 477c9058..f5c69e32 100644 --- a/gns3server/schemas/__init__.py +++ b/gns3server/schemas/__init__.py @@ -24,6 +24,7 @@ from .controller.links import LinkCreate, LinkUpdate, Link from .controller.computes import ComputeCreate, ComputeUpdate, AutoIdlePC, Compute from .controller.templates import TemplateCreate, TemplateUpdate, TemplateUsage, Template from .controller.images import Image, ImageType +from .controller.appliances import Appliance from .controller.drawings import Drawing from .controller.gns3vm import GNS3VM from .controller.nodes import NodeCreate, NodeUpdate, NodeDuplicate, NodeCapture, Node diff --git a/gns3server/schemas/controller/appliances.py b/gns3server/schemas/controller/appliances.py new file mode 100644 index 00000000..604af3f7 --- /dev/null +++ b/gns3server/schemas/controller/appliances.py @@ -0,0 +1,463 @@ +# +# Copyright (C) 2021 GNS3 Technologies Inc. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from enum import Enum +from typing import List, Optional, Union +from uuid import UUID + +from pydantic import AnyUrl, BaseModel, EmailStr, Field, confloat, conint, constr + + +class Category(Enum): + + router = 'router' + multilayer_switch = 'multilayer_switch' + switch = 'switch' + firewall = 'firewall' + guest = 'guest' + + +class RegistryVersion(Enum): + + version1 = 1 + version2 = 2 + version3 = 3 + version4 = 4 + version5 = 5 + version6 = 6 + + +class Status(Enum): + + stable = 'stable' + experimental = 'experimental' + broken = 'broken' + + +class Availability(Enum): + + free = 'free' + with_registration = 'with-registration' + free_to_try = 'free-to-try' + service_contract = 'service-contract' + + +class ConsoleType(Enum): + + telnet = 'telnet' + vnc = 'vnc' + http = 'http' + https = 'https' + none = 'none' + + +class Docker(BaseModel): + + adapters: int = Field(..., title='Number of ethernet adapters') + image: str = Field(..., title='Docker image in the Docker Hub') + start_command: Optional[str] = Field( + None, + title='Command executed when the container start. Empty will use the default', + ) + environment: Optional[str] = Field(None, title='One KEY=VAR environment by line') + console_type: Optional[ConsoleType] = Field( + None, title='Type of console connection for the administration of the appliance' + ) + console_http_port: Optional[int] = Field( + None, description='Internal port in the container of the HTTP server' + ) + console_http_path: Optional[str] = Field( + None, description='Path of the web interface' + ) + extra_hosts: Optional[str] = Field( + None, description='Hosts which will be written to /etc/hosts into container' + ) + extra_volumes: Optional[List[str]] = Field( + None, + description='Additional directories to make persistent that are not included in the images VOLUME directive', + ) + + +class Iou(BaseModel): + + ethernet_adapters: int = Field(..., title='Number of ethernet adapters') + serial_adapters: int = Field(..., title='Number of serial adapters') + nvram: int = Field(..., title='Host NVRAM') + ram: int = Field(..., title='Host RAM') + startup_config: str = Field(..., title='Config loaded at startup') + + +class Chassis(Enum): + + chassis_1720 = '1720' + chassis_1721 = '1721' + chassis_1750 = '1750' + chassis_1751 = '1751' + chassis_1760 = '1760' + chassis_2610 = '2610' + chassis_2620 = '2620' + chassis_2610XM = '2610XM' + chassis_2620XM = '2620XM' + chassis_2650XM = '2650XM' + chassis_2621 = '2621' + chassis_2611XM = '2611XM' + chassis_2621XM = '2621XM' + chassis_2651XM = '2651XM' + chassis_3620 = '3620' + chassis_3640 = '3640' + chassis_3660 = '3660' + + +class Platform(Enum): + + c1700 = 'c1700' + c2600 = 'c2600' + c2691 = 'c2691' + c3725 = 'c3725' + c3745 = 'c3745' + c3600 = 'c3600' + c7200 = 'c7200' + + +class Midplane(Enum): + + std = 'std' + vxr = 'vxr' + + +class Npe(Enum): + + npe_100 = 'npe-100' + npe_150 = 'npe-150' + npe_175 = 'npe-175' + npe_200 = 'npe-200' + npe_225 = 'npe-225' + npe_300 = 'npe-300' + npe_400 = 'npe-400' + npe_g2 = 'npe-g2' + + +class AdapterType(Enum): + + e1000 = 'e1000' + e1000_82544gc = 'e1000-82544gc' + e1000_82545em = 'e1000-82545em' + e1000e = 'e1000e' + i82550 = 'i82550' + i82551 = 'i82551' + i82557a = 'i82557a' + i82557b = 'i82557b' + i82557c = 'i82557c' + i82558a = 'i82558a' + i82558b = 'i82558b' + i82559a = 'i82559a' + i82559b = 'i82559b' + i82559c = 'i82559c' + i82559er = 'i82559er' + i82562 = 'i82562' + i82801 = 'i82801' + ne2k_pci = 'ne2k_pci' + pcnet = 'pcnet' + rocker = 'rocker' + rtl8139 = 'rtl8139' + virtio = 'virtio' + virtio_net_pci = 'virtio-net-pci' + vmxnet3 = 'vmxnet3' + + +class DiskInterface(Enum): + + ide = 'ide' + sata = 'sata' + nvme = 'nvme' + scsi = 'scsi' + sd = 'sd' + mtd = 'mtd' + floppy = 'floppy' + pflash = 'pflash' + virtio = 'virtio' + none = 'none' + + +class Arch(Enum): + + aarch64 = 'aarch64' + alpha = 'alpha' + arm = 'arm' + cris = 'cris' + i386 = 'i386' + lm32 = 'lm32' + m68k = 'm68k' + microblaze = 'microblaze' + microblazeel = 'microblazeel' + mips = 'mips' + mips64 = 'mips64' + mips64el = 'mips64el' + mipsel = 'mipsel' + moxie = 'moxie' + or32 = 'or32' + ppc = 'ppc' + ppc64 = 'ppc64' + ppcemb = 'ppcemb' + s390x = 's390x' + sh4 = 'sh4' + sh4eb = 'sh4eb' + sparc = 'sparc' + sparc64 = 'sparc64' + tricore = 'tricore' + unicore32 = 'unicore32' + x86_64 = 'x86_64' + xtensa = 'xtensa' + xtensaeb = 'xtensaeb' + + +class ConsoleType1(Enum): + + telnet = 'telnet' + vnc = 'vnc' + spice = 'spice' + spice_agent = 'spice+agent' + none = 'none' + + +class BootPriority(Enum): + + c = 'c' + d = 'd' + n = 'n' + cn = 'cn' + cd = 'cd' + dn = 'dn' + dc = 'dc' + nc = 'nc' + nd = 'nd' + + +class Kvm(Enum): + + require = 'require' + allow = 'allow' + disable = 'disable' + + +class ProcessPriority(Enum): + + realtime = 'realtime' + very_high = 'very high' + high = 'high' + normal = 'normal' + low = 'low' + very_low = 'very low' + null = 'null' + + +class Qemu(BaseModel): + + adapter_type: AdapterType = Field(..., title='Type of network adapter') + adapters: int = Field(..., title='Number of adapters') + ram: int = Field(..., title='Ram allocated to the appliance (MB)') + cpus: Optional[int] = Field(None, title='Number of Virtual CPU') + hda_disk_interface: Optional[DiskInterface] = Field( + None, title='Disk interface for the installed hda_disk_image' + ) + hdb_disk_interface: Optional[DiskInterface] = Field( + None, title='Disk interface for the installed hdb_disk_image' + ) + hdc_disk_interface: Optional[DiskInterface] = Field( + None, title='Disk interface for the installed hdc_disk_image' + ) + hdd_disk_interface: Optional[DiskInterface] = Field( + None, title='Disk interface for the installed hdd_disk_image' + ) + arch: Arch = Field(..., title='Architecture emulated') + console_type: ConsoleType1 = Field( + ..., title='Type of console connection for the administration of the appliance' + ) + boot_priority: Optional[BootPriority] = Field( + None, + title='Disk boot priority. Refer to -boot option in qemu manual for more details.', + ) + kernel_command_line: Optional[str] = Field( + None, title='Command line parameters send to the kernel' + ) + kvm: Kvm = Field(..., title='KVM requirements') + options: Optional[str] = Field( + None, title='Optional additional qemu command line options' + ) + cpu_throttling: Optional[confloat(ge=0.0, le=100.0)] = Field( + None, title='Throttle the CPU' + ) + process_priority: Optional[ProcessPriority] = Field( + None, title='Process priority for QEMU' + ) + + +class Compression(Enum): + + bzip2 = 'bzip2' + gzip = 'gzip' + lzma = 'lzma' + xz = 'xz' + rar = 'rar' + zip = 'zip' + field_7z = '7z' + + +class Image(BaseModel): + + filename: str = Field(..., title='Filename') + version: str = Field(..., title='Version of the file') + md5sum: constr(regex=r'^[a-f0-9]{32}$') = Field(..., title='md5sum of the file') + filesize: int = Field(..., title='File size in bytes') + download_url: Optional[Union[AnyUrl, constr(max_length=0)]] = Field( + None, title='Download url where you can download the appliance from a browser' + ) + direct_download_url: Optional[Union[AnyUrl, constr(max_length=0)]] = Field( + None, + title='Optional. Non authenticated url to the image file where you can download the image.', + ) + compression: Optional[Compression] = Field( + None, title='Optional, compression type of direct download url image.' + ) + + +class Images(BaseModel): + + kernel_image: Optional[str] = Field(None, title='Kernel image') + initrd: Optional[str] = Field(None, title='Initrd disk image') + image: Optional[str] = Field(None, title='OS image') + bios_image: Optional[str] = Field(None, title='Bios image') + hda_disk_image: Optional[str] = Field(None, title='Hda disk image') + hdb_disk_image: Optional[str] = Field(None, title='Hdc disk image') + hdc_disk_image: Optional[str] = Field(None, title='Hdd disk image') + hdd_disk_image: Optional[str] = Field(None, title='Hdd diskimage') + cdrom_image: Optional[str] = Field(None, title='cdrom image') + + +class Version(BaseModel): + + name: str = Field(..., title='Name of the version') + idlepc: Optional[constr(regex=r'^0x[0-9a-f]{8}')] = None + images: Optional[Images] = Field(None, title='Images used for this version') + + +class DynamipsSlot(Enum): + + C7200_IO_2FE = 'C7200-IO-2FE' + C7200_IO_FE = 'C7200-IO-FE' + C7200_IO_GE_E = 'C7200-IO-GE-E' + NM_16ESW = 'NM-16ESW' + NM_1E = 'NM-1E' + NM_1FE_TX = 'NM-1FE-TX' + NM_4E = 'NM-4E' + NM_4T = 'NM-4T' + PA_2FE_TX = 'PA-2FE-TX' + PA_4E = 'PA-4E' + PA_4T_ = 'PA-4T+' + PA_8E = 'PA-8E' + PA_8T = 'PA-8T' + PA_A1 = 'PA-A1' + PA_FE_TX = 'PA-FE-TX' + PA_GE = 'PA-GE' + PA_POS_OC3 = 'PA-POS-OC3' + C2600_MB_2FE = 'C2600-MB-2FE' + C2600_MB_1E = 'C2600-MB-1E' + C1700_MB_1FE = 'C1700-MB-1FE' + C2600_MB_2E = 'C2600-MB-2E' + C2600_MB_1FE = 'C2600-MB-1FE' + C1700_MB_WIC1 = 'C1700-MB-WIC1' + GT96100_FE = 'GT96100-FE' + Leopard_2FE = 'Leopard-2FE' + _ = '' + + +class DynamipsWic(Enum): + + WIC_1ENET = 'WIC-1ENET' + WIC_1T = 'WIC-1T' + WIC_2T = 'WIC-2T' + + +class Dynamips(BaseModel): + + chassis: Optional[Chassis] = Field(None, title='Chassis type') + platform: Platform = Field(..., title='Platform type') + ram: conint(ge=1) = Field(..., title='Amount of ram') + nvram: conint(ge=1) = Field(..., title='Amount of nvram') + startup_config: Optional[str] = Field(None, title='Config loaded at startup') + wic0: Optional[DynamipsWic] = None + wic1: Optional[DynamipsWic] = None + wic2: Optional[DynamipsWic] = None + slot0: Optional[DynamipsSlot] = None + slot1: Optional[DynamipsSlot] = None + slot2: Optional[DynamipsSlot] = None + slot3: Optional[DynamipsSlot] = None + slot4: Optional[DynamipsSlot] = None + slot5: Optional[DynamipsSlot] = None + slot6: Optional[DynamipsSlot] = None + midplane: Optional[Midplane] = None + npe: Optional[Npe] = None + + +class Appliance(BaseModel): + + appliance_id: UUID = Field(..., title='Appliance ID') + name: str = Field(..., title='Appliance name') + category: Category = Field(..., title='Category of the appliance') + description: str = Field( + ..., title='Description of the appliance. Could be a marketing description' + ) + vendor_name: str = Field(..., title='Name of the vendor') + vendor_url: Union[AnyUrl, constr(max_length=0)] = Field(..., title='Website of the vendor') + documentation_url: Optional[Union[AnyUrl, constr(max_length=0)]] = Field( + None, + title='An optional documentation for using the appliance on vendor website', + ) + product_name: str = Field(..., title='Product name') + product_url: Optional[Union[AnyUrl, constr(max_length=0)]] = Field( + None, title='An optional product url on vendor website' + ) + registry_version: RegistryVersion = Field( + ..., title='Version of the registry compatible with this appliance' + ) + status: Status = Field(..., title='Document if the appliance is working or not') + availability: Optional[Availability] = Field( + None, + title='About image availability: can be downloaded directly; download requires a free registration; paid but a trial version (time or feature limited) is available; not available publicly', + ) + maintainer: str = Field(..., title='Maintainer name') + maintainer_email: Union[EmailStr, constr(max_length=0)] = Field(..., title='Maintainer email') + usage: Optional[str] = Field(None, title='How to use the appliance') + symbol: Optional[str] = Field(None, title='An optional symbol for the appliance') + first_port_name: Optional[str] = Field( + None, title='Optional name of the first networking port example: eth0' + ) + port_name_format: Optional[str] = Field( + None, title='Optional formating of the networking port example: eth{0}' + ) + port_segment_size: Optional[int] = Field( + None, + title='Optional port segment size. A port segment is a block of port. For example Ethernet0/0 Ethernet0/1 is the module 0 with a port segment size of 2', + ) + linked_clone: Optional[bool] = Field( + None, title="False if you don't want to use a single image for all nodes" + ) + docker: Optional[Docker] = Field(None, title='Docker specific options') + iou: Optional[Iou] = Field(None, title='IOU specific options') + dynamips: Optional[Dynamips] = Field(None, title='Dynamips specific options') + qemu: Optional[Qemu] = Field(None, title='Qemu specific options') + images: Optional[List[Image]] = Field(None, title='Images for this appliance') + versions: Optional[List[Version]] = Field(None, title='Versions of the appliance') diff --git a/tests/api/routes/controller/test_appliances.py b/tests/api/routes/controller/test_appliances.py index 53fcc73b..a57b04ad 100644 --- a/tests/api/routes/controller/test_appliances.py +++ b/tests/api/routes/controller/test_appliances.py @@ -33,10 +33,10 @@ class TestApplianceRoutes: assert response.status_code == status.HTTP_200_OK assert len(response.json()) > 0 - async def test_appliance_download(self, app: FastAPI, client: AsyncClient) -> None: + async def test_get_appliance(self, app: FastAPI, client: AsyncClient) -> None: appliance_id = "3bf492b6-5717-4257-9bfd-b34617c6f133" # Cisco IOSv appliance - response = await client.get(app.url_path_for("download_appliance", appliance_id=appliance_id)) + response = await client.get(app.url_path_for("get_appliance", appliance_id=appliance_id)) assert response.status_code == status.HTTP_200_OK assert response.json()["appliance_id"] == appliance_id From 01da25a7c7bb88a62410916944475d0ac0c4261a Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 18 Oct 2021 21:53:29 +1030 Subject: [PATCH 3/6] Fix appliance validation with Pydantic --- gns3server/schemas/controller/appliances.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gns3server/schemas/controller/appliances.py b/gns3server/schemas/controller/appliances.py index 604af3f7..ceb96459 100644 --- a/gns3server/schemas/controller/appliances.py +++ b/gns3server/schemas/controller/appliances.py @@ -14,10 +14,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +# Generated from JSON schema using https://github.com/koxudaxi/datamodel-code-generator + +from __future__ import annotations + from enum import Enum from typing import List, Optional, Union from uuid import UUID - from pydantic import AnyUrl, BaseModel, EmailStr, Field, confloat, conint, constr From 88d98cf02eda5adb5ccd9c3a8b9090fc4cb0d10d Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 18 Oct 2021 22:12:10 +1030 Subject: [PATCH 4/6] Fix tests and workaround issue with flake8 --- gns3server/controller/appliance_manager.py | 11 +++++------ gns3server/schemas/controller/appliances.py | 4 ++-- tests/controller/test_controller.py | 5 ++--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/gns3server/controller/appliance_manager.py b/gns3server/controller/appliance_manager.py index c00b26fd..eb968f0d 100644 --- a/gns3server/controller/appliance_manager.py +++ b/gns3server/controller/appliance_manager.py @@ -298,19 +298,18 @@ class ApplianceManager: path = os.path.join(directory, file) try: with open(path, encoding="utf-8") as f: - json_data = json.load(f) - schemas.Appliance.parse_obj(json_data) - appliance = Appliance(path, json_data, builtin=builtin) - appliance_data = appliance.asdict() # Check if loaded without error + appliance = Appliance(path, json.load(f), builtin=builtin) + json_data = appliance.asdict() # Check if loaded without error if appliance.status != "broken": + schemas.Appliance.parse_obj(json_data) self._appliances[appliance.id] = appliance if not appliance.symbol or appliance.symbol.startswith(":/symbols/"): # apply a default symbol if the appliance has none or a default symbol - default_symbol = self._get_default_symbol(appliance_data, symbol_theme) + default_symbol = self._get_default_symbol(json_data, symbol_theme) if default_symbol: appliance.symbol = default_symbol except (ValueError, OSError, KeyError, ValidationError) as e: - log.warning(f"Cannot load appliance file '{path}': {e}") + print(f"Cannot load appliance file '{path}': {e}") continue def _get_default_symbol(self, appliance: dict, symbol_theme: str) -> str: diff --git a/gns3server/schemas/controller/appliances.py b/gns3server/schemas/controller/appliances.py index ceb96459..e6534736 100644 --- a/gns3server/schemas/controller/appliances.py +++ b/gns3server/schemas/controller/appliances.py @@ -323,7 +323,7 @@ class Image(BaseModel): filename: str = Field(..., title='Filename') version: str = Field(..., title='Version of the file') - md5sum: constr(regex=r'^[a-f0-9]{32}$') = Field(..., title='md5sum of the file') + md5sum: str = Field(..., title='md5sum of the file', regex='^[a-f0-9]{32}$') filesize: int = Field(..., title='File size in bytes') download_url: Optional[Union[AnyUrl, constr(max_length=0)]] = Field( None, title='Download url where you can download the appliance from a browser' @@ -353,7 +353,7 @@ class Images(BaseModel): class Version(BaseModel): name: str = Field(..., title='Name of the version') - idlepc: Optional[constr(regex=r'^0x[0-9a-f]{8}')] = None + idlepc: Optional[str] = Field(None, regex='^0x[0-9a-f]{8}') images: Optional[Images] = Field(None, title='Images used for this version') diff --git a/tests/controller/test_controller.py b/tests/controller/test_controller.py index 4b26b054..9f2060e9 100644 --- a/tests/controller/test_controller.py +++ b/tests/controller/test_controller.py @@ -385,14 +385,13 @@ def test_appliances(controller, config, tmpdir): for appliance in controller.appliance_manager.appliances.values(): assert appliance.asdict()["status"] != "broken" assert "Alpine Linux" in [c.asdict()["name"] for c in controller.appliance_manager.appliances.values()] - assert "My Appliance" in [c.asdict()["name"] for c in controller.appliance_manager.appliances.values()] + assert "My Appliance" not in [c.asdict()["name"] for c in controller.appliance_manager.appliances.values()] for c in controller.appliance_manager.appliances.values(): j = c.asdict() if j["name"] == "Alpine Linux": assert j["builtin"] - elif j["name"] == "My Appliance": - assert not j["builtin"] + @pytest.mark.asyncio async def test_autoidlepc(controller): From be473aaaf740c261d651c183ff3244ac0c715d3b Mon Sep 17 00:00:00 2001 From: grossmj Date: Mon, 18 Oct 2021 22:16:57 +1030 Subject: [PATCH 5/6] Remove from __future__ import annotations --- gns3server/schemas/controller/appliances.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gns3server/schemas/controller/appliances.py b/gns3server/schemas/controller/appliances.py index e6534736..b1f0d61c 100644 --- a/gns3server/schemas/controller/appliances.py +++ b/gns3server/schemas/controller/appliances.py @@ -16,8 +16,6 @@ # Generated from JSON schema using https://github.com/koxudaxi/datamodel-code-generator -from __future__ import annotations - from enum import Enum from typing import List, Optional, Union from uuid import UUID From a31e5615a47e6a60cef40b3bf09c9c4ea4e7843e Mon Sep 17 00:00:00 2001 From: grossmj Date: Tue, 19 Oct 2021 15:15:10 +1030 Subject: [PATCH 6/6] Add a custom version to an appliance --- .../api/routes/controller/appliances.py | 32 ++++++++++++++++++- gns3server/schemas/__init__.py | 2 +- gns3server/schemas/controller/appliances.py | 12 +++---- .../api/routes/controller/test_appliances.py | 25 +++++++++++++++ 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/gns3server/api/routes/controller/appliances.py b/gns3server/api/routes/controller/appliances.py index 18532e05..550473e1 100644 --- a/gns3server/api/routes/controller/appliances.py +++ b/gns3server/api/routes/controller/appliances.py @@ -26,7 +26,12 @@ from uuid import UUID from gns3server import schemas from gns3server.controller import Controller -from gns3server.controller.controller_error import ControllerNotFoundError +from gns3server.controller.controller_error import ( + ControllerError, + ControllerBadRequestError, + ControllerNotFoundError +) + from gns3server.db.repositories.images import ImagesRepository from gns3server.db.repositories.templates import TemplatesRepository from gns3server.db.repositories.rbac import RbacRepository @@ -68,6 +73,31 @@ def get_appliance(appliance_id: UUID) -> schemas.Appliance: return appliance.asdict() +@router.post("/{appliance_id}/version", status_code=status.HTTP_201_CREATED) +def add_appliance_version(appliance_id: UUID, appliance_version: schemas.ApplianceVersion) -> schemas.Appliance: + """ + Add a version to an appliance + """ + + controller = Controller.instance() + appliance = controller.appliance_manager.appliances.get(str(appliance_id)) + if not appliance: + raise ControllerNotFoundError(message=f"Could not find appliance '{appliance_id}'") + + if not appliance.versions: + raise ControllerBadRequestError(message=f"Appliance '{appliance_id}' do not have versions") + + if not appliance_version.images: + raise ControllerBadRequestError(message=f"Version '{appliance_version.name}' must contain images") + + for version in appliance.versions: + if version.get("name") == appliance_version.name: + raise ControllerError(message=f"Appliance '{appliance_id}' already has version '{appliance_version.name}'") + + appliance.versions.append(appliance_version.dict(exclude_unset=True)) + return appliance.asdict() + + @router.post("/{appliance_id}/install", status_code=status.HTTP_204_NO_CONTENT) async def install_appliance( appliance_id: UUID, diff --git a/gns3server/schemas/__init__.py b/gns3server/schemas/__init__.py index f5c69e32..5a3e99a3 100644 --- a/gns3server/schemas/__init__.py +++ b/gns3server/schemas/__init__.py @@ -24,7 +24,7 @@ from .controller.links import LinkCreate, LinkUpdate, Link from .controller.computes import ComputeCreate, ComputeUpdate, AutoIdlePC, Compute from .controller.templates import TemplateCreate, TemplateUpdate, TemplateUsage, Template from .controller.images import Image, ImageType -from .controller.appliances import Appliance +from .controller.appliances import ApplianceVersion, Appliance from .controller.drawings import Drawing from .controller.gns3vm import GNS3VM from .controller.nodes import NodeCreate, NodeUpdate, NodeDuplicate, NodeCapture, Node diff --git a/gns3server/schemas/controller/appliances.py b/gns3server/schemas/controller/appliances.py index b1f0d61c..5af72108 100644 --- a/gns3server/schemas/controller/appliances.py +++ b/gns3server/schemas/controller/appliances.py @@ -317,7 +317,7 @@ class Compression(Enum): field_7z = '7z' -class Image(BaseModel): +class ApplianceImage(BaseModel): filename: str = Field(..., title='Filename') version: str = Field(..., title='Version of the file') @@ -335,7 +335,7 @@ class Image(BaseModel): ) -class Images(BaseModel): +class ApplianceVersionImages(BaseModel): kernel_image: Optional[str] = Field(None, title='Kernel image') initrd: Optional[str] = Field(None, title='Initrd disk image') @@ -348,11 +348,11 @@ class Images(BaseModel): cdrom_image: Optional[str] = Field(None, title='cdrom image') -class Version(BaseModel): +class ApplianceVersion(BaseModel): name: str = Field(..., title='Name of the version') idlepc: Optional[str] = Field(None, regex='^0x[0-9a-f]{8}') - images: Optional[Images] = Field(None, title='Images used for this version') + images: Optional[ApplianceVersionImages] = Field(None, title='Images used for this version') class DynamipsSlot(Enum): @@ -460,5 +460,5 @@ class Appliance(BaseModel): iou: Optional[Iou] = Field(None, title='IOU specific options') dynamips: Optional[Dynamips] = Field(None, title='Dynamips specific options') qemu: Optional[Qemu] = Field(None, title='Qemu specific options') - images: Optional[List[Image]] = Field(None, title='Images for this appliance') - versions: Optional[List[Version]] = Field(None, title='Versions of the appliance') + images: Optional[List[ApplianceImage]] = Field(None, title='Images for this appliance') + versions: Optional[List[ApplianceVersion]] = Field(None, title='Versions of the appliance') diff --git a/tests/api/routes/controller/test_appliances.py b/tests/api/routes/controller/test_appliances.py index a57b04ad..089466c2 100644 --- a/tests/api/routes/controller/test_appliances.py +++ b/tests/api/routes/controller/test_appliances.py @@ -66,3 +66,28 @@ class TestApplianceRoutes: appliance_id = "1cfdf900-7c30-4cb7-8f03-3f61d2581633" # Empty VM appliance response = await client.post(app.url_path_for("install_appliance", appliance_id=appliance_id)) assert response.status_code == status.HTTP_400_BAD_REQUEST + + async def test_add_version_appliance(self, app: FastAPI, client: AsyncClient) -> None: + + appliance_id = "1cfdf900-7c30-4cb7-8f03-3f61d2581633" # Empty VM appliance + new_version = { + "name": "99G", + "images": { + "hda_disk_image": "empty99G.qcow2" + } + } + response = await client.post(app.url_path_for("add_appliance_version", appliance_id=appliance_id), json=new_version) + assert response.status_code == status.HTTP_201_CREATED + assert new_version in response.json()["versions"] + + async def test_add_existing_version_appliance(self, app: FastAPI, client: AsyncClient) -> None: + + appliance_id = "1cfdf900-7c30-4cb7-8f03-3f61d2581633" # Empty VM appliance + new_version = { + "name": "8G", + "images": { + "hda_disk_image": "empty8G.qcow2" + } + } + response = await client.post(app.url_path_for("add_appliance_version", appliance_id=appliance_id), json=new_version) + assert response.status_code == status.HTTP_409_CONFLICT