Embed the appliances in the server. (#927)

This add a /appliances call

docs/curl.rst

@@ -74,10 +74,6 @@ With this project id we can now create two VPCS Node.
         "node_id": "f124dec0-830a-451e-a314-be50bbd58a00",
         "node_type": "vpcs",
         "project_id": "b8c070f7-f34c-4b7b-ba6f-be3d26ed073f",
-        "properties": {
-            "startup_script": null,
-            "startup_script_path": null
-        },
         "status": "stopped"
     }
 
@@ -91,10 +87,6 @@ With this project id we can now create two VPCS Node.
         "node_id": "83892a4d-aea0-4350-8b3e-d0af3713da74",
         "node_type": "vpcs",
         "project_id": "b8c070f7-f34c-4b7b-ba6f-be3d26ed073f",
-        "properties": {
-            "startup_script": null,
-            "startup_script_path": null
-        },
         "status": "stopped"
     }
 
@@ -230,8 +222,52 @@ This will display a red square in the middle of your topologies:
 Tips: you can embed png/jpg... by using a base64 encoding in the SVG.
 
 
-Create a Qemu node
-###################
+Creation of nodes
+#################
+
+Their is two way of adding nodes. Manual by passing all the information require for a Node.
+
+Or by using an appliance. The appliance is a node model saved in your server.
+
+Using an appliance
+------------------
+
+First you need to list the available appliances
+
+.. code-block:: shell-session
+
+    # curl "http://localhost:3080/v2/appliances"
+
+    [
+        {
+            "appliance_id": "5fa8a8ca-0f80-4ac4-8104-2b32c7755443",
+            "category": "guest",
+            "compute_id": "vm",
+            "default_name_format": "{name}-{0}",
+            "name": "MicroCore",
+            "node_type": "qemu",
+            "symbol": ":/symbols/qemu_guest.svg"
+        },
+        {
+            "appliance_id": "9cd59d5a-c70f-4454-8313-6a9e81a8278f",
+            "category": "guest",
+            "compute_id": "vm",
+            "default_name_format": "{name}-{0}",
+            "name": "Chromium",
+            "node_type": "docker",
+            "symbol": ":/symbols/docker_guest.svg"
+        }
+    ]
+
+Now you can use the appliance and put it at a specific position
+
+.. code-block:: shell-session
+
+ # curl -X POST http://localhost:3080/v2/projects/b8c070f7-f34c-4b7b-ba6f-be3d26ed073f -d '{"x": 12, "y": 42}'
+
+
+Manual creation of a Qemu node
+-------------------------------
 
 .. code-block:: shell-session
 
@@ -315,9 +351,8 @@ Create a Qemu node
     }
 
 
-Create a dynamips node
-######################
-
+Manual creation of a dynamips node
+-----------------------------------
 
 .. code-block:: shell-session
 

docs/glossary.rst

@@ -1,11 +1,29 @@
 Glossary
 ========
 
+Topology
+--------
+
+The place where you have all things (node, drawing, link...)
+
+
 Node
 -----
 
 A Virtual Machine (Dynamips, IOU, Qemu, VPCS...), a cloud, a builtin device (switch, hub...)
 
+Appliance
+---------
+
+A model for a node. When you drag an appliance to the topology a node is created.
+
+
+Appliance template
+------------------
+
+A file (.gns3a) use for creating new node model.
+
+
 Drawing
 --------
 

gns3server/controller/__init__.py

@@ -57,13 +57,12 @@ class Controller:
 
         # Store settings shared by the different GUI will be replace by dedicated API later
         self._settings = None
-        self._local_server = None
+        self._appliances = {}
+        self._appliance_templates = {}
+
         self._config_file = os.path.join(Config.instance().config_dir, "gns3_controller.conf")
         log.info("Load controller configuration file {}".format(self._config_file))
 
-        self._appliance_templates = {}
-        self.load_appliances()
-
     def load_appliances(self):
         self._appliance_templates = {}
         for file in os.listdir(get_resource('appliances')):
@@ -72,6 +71,59 @@ class Controller:
             if appliance.status != 'broken':
                 self._appliance_templates[appliance.id] = appliance
 
+        self._appliances = {}
+        for vm in self._settings.get("Qemu", {}).get("vms", []):
+            vm["node_type"] = "qemu"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("IOU", {}).get("devices", []):
+            vm["node_type"] = "iou"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("Docker", {}).get("containers", []):
+            vm["node_type"] = "docker"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("Builtin", {}).get("cloud_nodes", []):
+            vm["node_type"] = "cloud"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("Builtin", {}).get("ethernet_switches", []):
+            vm["node_type"] = "ethernet_switch"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("Builtin", {}).get("ethernet_hubs", []):
+            vm["node_type"] = "ethernet_hub"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("Dynamips", {}).get("routers", []):
+            vm["node_type"] = "dynamips"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("VMware", {}).get("vms", []):
+            vm["node_type"] = "vmware"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("VirtualBox", {}).get("vms", []):
+            vm["node_type"] = "virtualbox"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        for vm in self._settings.get("VPCS", {}).get("nodes", []):
+            vm["node_type"] = "vpcs"
+            appliance = Appliance(None, vm)
+            self._appliances[appliance.id] = appliance
+        # Add builtins
+        builtins = []
+        builtins.append(Appliance(None, {"node_type": "cloud", "name": "Cloud", "category": 2, "symbol": ":/symbols/cloud.svg"}, builtin=True))
+        builtins.append(Appliance(None, {"node_type": "nat", "name": "NAT", "category": 2, "symbol": ":/symbols/cloud.svg"}, builtin=True))
+        builtins.append(Appliance(None, {"node_type": "vpcs", "name": "VPCS", "category": 2, "symbol": ":/symbols/vpcs_guest.svg"}, builtin=True))
+        builtins.append(Appliance(None, {"node_type": "ethernet_switch", "name": "Ethernet switch", "category": 1, "symbol": ":/symbols/ethernet_switch.svg"}, builtin=True))
+        builtins.append(Appliance(None, {"node_type": "ethernet_hub", "name": "Ethernet hub", "category": 1, "symbol": ":/symbols/hub.svg"}, builtin=True))
+        builtins.append(Appliance(None, {"node_type": "frame_relay_switch", "name": "Frame Relay switch", "category": 1, "symbol": ":/symbols/frame_relay_switch.svg"}, builtin=True))
+        builtins.append(Appliance(None, {"node_type": "atm_switch", "name": "ATM switch", "category": 1, "symbol": ":/symbols/atm_switch.svg"}, builtin=True))
+        for b in builtins:
+            self._appliances[b.id] = b
+
     @asyncio.coroutine
     def start(self):
         log.info("Start controller")
@@ -190,6 +242,7 @@ class Controller:
         if "gns3vm" in data:
             self.gns3vm.settings = data["gns3vm"]
 
+        self.load_appliances()
         return data["computes"]
 
     @asyncio.coroutine
@@ -309,6 +362,7 @@ class Controller:
         self._settings = val
         self._settings["modification_uuid"] = str(uuid.uuid4())  # We add a modification id to the settings it's help the gui to detect changes
         self.save()
+        self.load_appliances()
         self.notification.emit("settings.updated", val)
 
     @asyncio.coroutine
@@ -515,6 +569,13 @@ class Controller:
         """
         return self._appliance_templates
 
+    @property
+    def appliances(self):
+        """
+        :returns: The dictionary of appliances managed by GNS3
+        """
+        return self._appliances
+
     def projects_directory(self):
         server_config = Config.instance().get_section_config("Server")
         return os.path.expanduser(server_config.get("projects_path", "~/GNS3/projects"))

gns3server/controller/appliance.py

@@ -18,21 +18,56 @@
 import uuid
 
 
+# Convert old GUI category to text category
+ID_TO_CATEGORY = {
+    3: "firewall",
+    2: "guest",
+    1: "switch",
+    0: "router"
+}
+
+
 class Appliance:
 
-    def __init__(self, appliance_id, data):
+    def __init__(self, appliance_id, data, builtin=False):
         if appliance_id is None:
             self._id = str(uuid.uuid4())
         else:
             self._id = appliance_id
         self._data = data
+        self._builtin = builtin
 
     @property
     def id(self):
         return self._id
 
+    @property
+    def data(self):
+        return self._data
+
+    @property
+    def name(self):
+        return self._data["name"]
+
+    @property
+    def compute_id(self):
+        return self._data.get("server")
+
+    @property
+    def builtin(self):
+        return self._builtin
+
     def __json__(self):
         """
         Appliance data (a hash)
         """
-        return self._data
+        return {
+            "appliance_id": self._id,
+            "node_type": self._data["node_type"],
+            "name": self._data["name"],
+            "default_name_format": self._data.get("default_name_format", "{name}-{0}"),
+            "category": ID_TO_CATEGORY[self._data["category"]],
+            "symbol": self._data["symbol"],
+            "compute_id": self.compute_id,
+            "builtin": self._builtin
+        }

gns3server/controller/project.py

@@ -19,6 +19,7 @@ import re
 import os
 import json
 import uuid
+import copy
 import shutil
 import asyncio
 import aiohttp
@@ -317,6 +318,29 @@ class Project:
             return self.update_allocated_node_name(new_name)
         return new_name
 
+    @open_required
+    @asyncio.coroutine
+    def add_node_from_appliance(self, appliance_id, x=0, y=0, compute_id=None):
+        """
+        Create a node from an appliance
+        """
+        try:
+            template = copy.copy(self.controller.appliances[appliance_id].data)
+        except KeyError:
+            msg = "Appliance {} doesn't exist".format(appliance_id)
+            log.error(msg)
+            raise aiohttp.web.HTTPNotFound(text=msg)
+        template["x"] = x
+        template["y"] = y
+        node_type = template.pop("node_type")
+        compute = self.controller.get_compute(template.pop("server", compute_id))
+        name = template.pop("name")
+        default_name_format = template.pop("default_name_format", "{name}-{0}")
+        name = default_name_format.replace("{name}", name)
+        node_id = str(uuid.uuid4())
+        node = yield from self.add_node(compute, name, node_id, node_type=node_type, **template)
+        return node
+
     @open_required
     @asyncio.coroutine
     def add_node(self, compute, name, node_id, dump=True, node_type=None, **kwargs):

gns3server/handlers/api/controller/appliance_handler.py

@@ -17,6 +17,9 @@
 
 from gns3server.web.route import Route
 from gns3server.controller import Controller
+from gns3server.schemas.node import NODE_OBJECT_SCHEMA
+from gns3server.schemas.appliance import APPLIANCE_USAGE_SCHEMA
+
 
 import logging
 log = logging.getLogger(__name__)
@@ -27,6 +30,17 @@ class ApplianceHandler:
 
     @Route.get(
         r"/appliances/templates",
+        description="List of appliance templates",
+        status_codes={
+            200: "Appliance template list returned"
+        })
+    def list_templates(request, response):
+
+        controller = Controller.instance()
+        response.json([c for c in controller.appliance_templates.values()])
+
+    @Route.get(
+        r"/appliances",
         description="List of appliance",
         status_codes={
             200: "Appliance list returned"
@@ -34,4 +48,27 @@ class ApplianceHandler:
     def list(request, response):
 
         controller = Controller.instance()
-        response.json([c for c in controller.appliance_templates.values()])
+        response.json([c for c in controller.appliances.values()])
+
+    @Route.post(
+        r"/projects/{project_id}/appliances/{appliance_id}",
+        description="Create a node from an appliance",
+        parameters={
+            "project_id": "Project UUID",
+            "appliance_id": "Appliance template UUID"
+        },
+        status_codes={
+            201: "Node created",
+            404: "The project or template doesn't exist"
+        },
+        input=APPLIANCE_USAGE_SCHEMA,
+        output=NODE_OBJECT_SCHEMA)
+    def create_node_from_appliance(request, response):
+
+        controller = Controller.instance()
+        project = controller.get_project(request.match_info["project_id"])
+        yield from project.add_node_from_appliance(request.match_info["appliance_id"],
+                                                   x=request.json["x"],
+                                                   y=request.json["y"],
+                                                   compute_id=request.json.get("compute_id"))
+        response.set_status(201)

gns3server/schemas/appliance.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+APPLIANCE_USAGE_SCHEMA = {
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "description": "Request validation to use an Appliance instance",
+    "type": "object",
+    "properties": {
+        "x": {
+            "description": "X position",
+            "type": "integer"
+        },
+        "y": {
+            "description": "Y position",
+            "type": "integer"
+        },
+        "compute_id": {
+            "description": "If the appliance don't have a default compute use this compute",
+            "type": ["null", "string"]
+        }
+    },
+    "additionalProperties": False,
+    "required": ["x", "y"]
+}

tests/controller/test_appliance.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2016 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from gns3server.controller.appliance import Appliance
+
+
+def test_appliance_json():
+    a = Appliance(None, {
+        "node_type": "qemu",
+        "name": "Test",
+        "default_name_format": "{name}-{0}",
+        "category": 0,
+        "symbol": "qemu.svg",
+        "server": "local"
+    })
+    assert a.__json__() == {
+        "appliance_id": a.id,
+        "node_type": "qemu",
+        "builtin": False,
+        "name": "Test",
+        "default_name_format": "{name}-{0}",
+        "category": "router",
+        "symbol": "qemu.svg",
+        "compute_id": "local"
+    }

tests/controller/test_controller.py

@@ -53,7 +53,7 @@ def test_load_controller_settings(controller, controller_config_path, async_run)
             "compute_id": "test1"
         }
     ]
-    data["settings"] = {"IOU": True}
+    data["settings"] = {"IOU": {"test": True}}
     data["gns3vm"] = {"vmname": "Test VM"}
     with open(controller_config_path, "w+") as f:
         json.dump(data, f)
@@ -468,3 +468,23 @@ def test_load_base_files(controller, config, tmpdir):
     # Check is the file has not been overwrite
     with open(str(tmpdir / 'iou_l2_base_startup-config.txt')) as f:
         assert f.read() == 'test'
+
+
+def test_appliance_templates(controller, async_run):
+    controller.load_appliances()
+    assert len(controller.appliance_templates) > 0
+
+
+def test_load_appliances(controller):
+    controller._settings = {
+        "Qemu": {
+            "vms": [
+                {
+                    "name": "Test"
+                }
+            ]
+        }
+    }
+    controller.load_appliances()
+    assert "Test" in [appliance.name for appliance in controller.appliances.values()]
+    assert "Cloud" in [appliance.name for appliance in controller.appliances.values()]

tests/controller/test_project.py

@@ -18,17 +18,15 @@
 
 import os
 import sys
-import uuid
-import json
 import pytest
 import aiohttp
-import zipfile
 from unittest.mock import MagicMock
 from tests.utils import AsyncioMagicMock, asyncio_patch
 from unittest.mock import patch
 from uuid import uuid4
 
 from gns3server.controller.project import Project
+from gns3server.controller.appliance import Appliance
 from gns3server.controller.ports.ethernet_port import EthernetPort
 from gns3server.config import Config
 
@@ -182,6 +180,42 @@ def test_add_node_non_local(async_run, controller):
     controller.notification.emit.assert_any_call("node.created", node.__json__())
 
 
+def test_add_node_from_appliance(async_run, controller):
+    """
+    For a local server we send the project path
+    """
+    compute = MagicMock()
+    compute.id = "local"
+    project = Project(controller=controller, name="Test")
+    controller._notification = MagicMock()
+    controller._appliances["fakeid"] = Appliance("fakeid", {
+        "server": "local",
+        "name": "Test",
+        "default_name_format": "{name}-{0}",
+        "node_type": "vpcs"
+
+    })
+    controller._computes["local"] = compute
+
+    response = MagicMock()
+    response.json = {"console": 2048}
+    compute.post = AsyncioMagicMock(return_value=response)
+
+    node = async_run(project.add_node_from_appliance("fakeid", x=23, y=12))
+
+    compute.post.assert_any_call('/projects', data={
+        "name": project._name,
+        "project_id": project._id,
+        "path": project._path
+    })
+    compute.post.assert_any_call('/projects/{}/vpcs/nodes'.format(project.id),
+                                 data={'node_id': node.id,
+                                       'name': 'Test-1'},
+                                 timeout=120)
+    assert compute in project._project_created_on_compute
+    controller.notification.emit.assert_any_call("node.created", node.__json__())
+
+
 def test_create_iou_on_multiple_node(async_run, controller):
     """
     Due to mac address collision you can't create an IOU node

tests/handlers/api/controller/test_appliance.py

@@ -16,10 +16,73 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 
+import uuid
+import pytest
+from unittest.mock import MagicMock
+from tests.utils import asyncio_patch
+
+
+from gns3server.controller import Controller
+from gns3server.controller.appliance import Appliance
+
+
+@pytest.fixture
+def compute(http_controller, async_run):
+    compute = MagicMock()
+    compute.id = "example.com"
+    compute.host = "example.org"
+    Controller.instance()._computes = {"example.com": compute}
+    return compute
+
+
+@pytest.fixture
+def project(http_controller, async_run):
+    return async_run(Controller.instance().add_project(name="Test"))
+
+
 def test_appliance_list(http_controller, controller):
 
-    response = http_controller.get("/appliances/templates")
+    id = str(uuid.uuid4())
+    controller.load_appliances()
+    controller._appliances[id] = Appliance(id, {
+        "node_type": "qemu",
+        "category": 0,
+        "name": "test",
+        "symbol": "guest.svg",
+        "default_name_format": "{name}-{0}",
+        "server": "local"
+    })
+    response = http_controller.get("/appliances", example=True)
     assert response.status == 200
-    assert response.route == "/appliances/templates"
+    assert response.route == "/appliances"
+    assert len(response.json) > 0
+
 
+def test_appliance_templates_list(http_controller, controller, async_run):
+
+    controller.load_appliances()
+    response = http_controller.get("/appliances/templates", example=True)
+    assert response.status == 200
     assert len(response.json) > 0
+
+
+def test_create_node_from_appliance(http_controller, controller, project, compute):
+
+    id = str(uuid.uuid4())
+    controller._appliances = {id: Appliance(id, {
+        "node_type": "qemu",
+        "category": 0,
+        "name": "test",
+        "symbol": "guest.svg",
+        "default_name_format": "{name}-{0}",
+        "server": "example.com"
+    })}
+    with asyncio_patch("gns3server.controller.project.Project.add_node_from_appliance") as mock:
+        response = http_controller.post("/projects/{}/appliances/{}".format(project.id, id), {
+            "x": 42,
+            "y": 12
+        })
+    mock.assert_called_with(id, x=42, y=12, compute_id=None)
+    print(response.body)
+    assert response.route == "/projects/{project_id}/appliances/{appliance_id}"
+    assert response.status == 201