Move endpoints to routes & preparations to use a database.

.gitignore

@@ -5,6 +5,9 @@ __pycache__
 #py.test
 .cache
 
+# environment file
+.env
+
 # C extensions
 *.so
 

gns3server/api/middlewares.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2020 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import time
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+from gns3server.version import __version__
+
+
+class AddExtraHeadersMiddleware:
+
+    def __init__(self, app: ASGIApp) -> None:
+        self.app = app
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+
+
+        if scope["type"] in ("http", "websocket") and scope["scheme"] in ("http", "ws"):
+            url = URL(scope=scope)
+            redirect_scheme = {"http": "https", "ws": "wss"}[url.scheme]
+            netloc = url.hostname if url.port in (80, 443) else url.netloc
+            url = url.replace(scheme=redirect_scheme, netloc=netloc)
+            response = RedirectResponse(url, status_code=307)
+            await response(scope, receive, send)
+
+
+@app.middleware("http")
+async def add_extra_headers(request: Request, call_next):
+    start_time = time.time()
+    response = await call_next(request)
+    process_time = time.time() - start_time
+    response.headers["X-Process-Time"] = str(process_time)
+    response.headers["X-GNS3-Server-Version"] = "{}".format(__version__)
+    return response

gns3server/api/routes/compute/atm_switch_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for ATM switch nodes.
+API routes for ATM switch nodes.
 """
 
 import os

gns3server/api/routes/compute/capabilities.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for capabilities
+API routes for capabilities
 """
 
 import sys

gns3server/api/routes/compute/cloud_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for cloud nodes.
+API routes for cloud nodes.
 """
 
 import os

gns3server/api/routes/compute/compute.py

@@ -17,7 +17,7 @@
 
 
 """
-API endpoints for compute.
+API routes for compute.
 """
 
 import os

gns3server/api/routes/compute/docker_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for Docker nodes.
+API routes for Docker nodes.
 """
 
 import os

gns3server/api/routes/compute/dynamips_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for Dynamips nodes.
+API routes for Dynamips nodes.
 """
 
 import os

gns3server/api/routes/compute/ethernet_hub_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for Ethernet hub nodes.
+API routes for Ethernet hub nodes.
 """
 
 import os

gns3server/api/routes/compute/ethernet_switch_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for Ethernet switch nodes.
+API routes for Ethernet switch nodes.
 """
 
 import os

gns3server/api/routes/compute/frame_relay_switch_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for Frame Relay switch nodes.
+API routes for Frame Relay switch nodes.
 """
 
 import os

gns3server/api/routes/compute/images.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for images.
+API routes for images.
 """
 
 import os

gns3server/api/routes/compute/iou_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for IOU nodes.
+API routes for IOU nodes.
 """
 
 import os

gns3server/api/routes/compute/nat_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for NAT nodes.
+API routes for NAT nodes.
 """
 
 import os

gns3server/api/routes/compute/notifications.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for compute notifications.
+API routes for compute notifications.
 """
 
 from fastapi import APIRouter, WebSocket, WebSocketDisconnect

gns3server/api/routes/compute/projects.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for projects.
+API routes for projects.
 """
 
 import os

gns3server/api/routes/compute/qemu_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for Qemu nodes.
+API routes for Qemu nodes.
 """
 
 import os

gns3server/api/routes/compute/virtualbox_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for VirtualBox nodes.
+API routes for VirtualBox nodes.
 """
 
 import os

gns3server/api/routes/compute/vmware_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for VMware nodes.
+API routes for VMware nodes.
 """
 
 import os

gns3server/api/routes/compute/vpcs_nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for VPCS nodes.
+API routes for VPCS nodes.
 """
 
 import os

gns3server/api/routes/controller/appliances.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for appliances.
+API routes for appliances.
 """
 
 from fastapi import APIRouter

gns3server/api/routes/controller/computes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for computes.
+API routes for computes.
 """
 
 from fastapi import APIRouter, status
@@ -118,7 +118,7 @@ async def get_images(compute_id: Union[str, UUID], emulator: str):
 async def forward_get(compute_id: Union[str, UUID], emulator: str, endpoint_path: str):
     """
     Forward a GET request to a compute.
-    Read the full compute API documentation for available endpoints.
+    Read the full compute API documentation for available routes.
     """
 
     compute = Controller.instance().get_compute(str(compute_id))
@@ -131,7 +131,7 @@ async def forward_get(compute_id: Union[str, UUID], emulator: str, endpoint_path
 async def forward_post(compute_id: Union[str, UUID], emulator: str, endpoint_path: str, compute_data: dict):
     """
     Forward a POST request to a compute.
-    Read the full compute API documentation for available endpoints.
+    Read the full compute API documentation for available routes.
     """
 
     compute = Controller.instance().get_compute(str(compute_id))
@@ -143,7 +143,7 @@ async def forward_post(compute_id: Union[str, UUID], emulator: str, endpoint_pat
 async def forward_put(compute_id: Union[str, UUID], emulator: str, endpoint_path: str, compute_data: dict):
     """
     Forward a PUT request to a compute.
-    Read the full compute API documentation for available endpoints.
+    Read the full compute API documentation for available routes.
     """
 
     compute = Controller.instance().get_compute(str(compute_id))

gns3server/api/routes/controller/drawings.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for drawings.
+API routes for drawings.
 """
 
 from fastapi import APIRouter, status

gns3server/api/routes/controller/gns3vm.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for managing the GNS3 VM.
+API routes for managing the GNS3 VM.
 """
 
 from fastapi import APIRouter

gns3server/api/routes/controller/links.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for links.
+API routes for links.
 """
 
 import multidict

gns3server/api/routes/controller/nodes.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for nodes.
+API routes for nodes.
 """
 
 import aiohttp

gns3server/api/routes/controller/notifications.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for controller notifications.
+API routes for controller notifications.
 """
 
 from fastapi import APIRouter, WebSocket, WebSocketDisconnect

gns3server/api/routes/controller/projects.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for projects.
+API routes for projects.
 """
 
 import os

gns3server/api/routes/controller/snapshots.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for snapshots.
+API routes for snapshots.
 """
 
 import logging

gns3server/api/routes/controller/symbols.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for symbols.
+API routes for symbols.
 """
 
 import os

gns3server/api/routes/controller/templates.py

@@ -16,7 +16,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 """
-API endpoints for templates.
+API routes for templates.
 """
 
 import hashlib

gns3server/api/server.py

@@ -19,8 +19,6 @@
 FastAPI app
 """
 
-import sys
-import asyncio
 import time
 
 from fastapi import FastAPI, Request
@@ -28,9 +26,7 @@ from starlette.exceptions import HTTPException as StarletteHTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 
-from gns3server.controller import Controller
+
-from gns3server.compute import MODULES
-from gns3server.compute.port_manager import PortManager
 from gns3server.controller.controller_error import (
     ControllerError,
     ControllerNotFoundError,
@@ -39,15 +35,15 @@ from gns3server.controller.controller_error import (
     ControllerUnauthorizedError
 )
 
-from gns3server.endpoints import controller
+from gns3server.api.routes import controller, index
-from gns3server.endpoints import index
+from gns3server.api.routes.compute import compute_api
-from gns3server.endpoints.compute import compute_api
+from gns3server.core import tasks
-from gns3server.utils.http_client import HTTPClient
 from gns3server.version import __version__
 
 import logging
 log = logging.getLogger(__name__)
 
+
 app = FastAPI(title="GNS3 controller API",
               description="This page describes the public controller API for GNS3",
               version="v3")
@@ -71,6 +67,8 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
+app.add_event_handler("startup", tasks.create_startup_handler(app))
+app.add_event_handler("shutdown", tasks.create_shutdown_handler(app))
 app.include_router(index.router, tags=["Index"])
 app.include_router(controller.router, prefix="/v3")
 app.mount("/v3/compute", compute_api)
@@ -138,58 +136,3 @@ async def add_extra_headers(request: Request, call_next):
     response.headers["X-Process-Time"] = str(process_time)
     response.headers["X-GNS3-Server-Version"] = "{}".format(__version__)
     return response
-
-
-@app.on_event("startup")
-async def startup_event():
-
-    loop = asyncio.get_event_loop()
-    logger = logging.getLogger("asyncio")
-    logger.setLevel(logging.ERROR)
-
-    if sys.platform.startswith("win"):
-
-        # Add a periodic callback to give a chance to process signals on Windows
-        # because asyncio.add_signal_handler() is not supported yet on that platform
-        # otherwise the loop runs outside of signal module's ability to trap signals.
-
-        def wakeup():
-            loop.call_later(0.5, wakeup)
-
-        loop.call_later(0.5, wakeup)
-
-    if log.getEffectiveLevel() == logging.DEBUG:
-        # On debug version we enable info that
-        # coroutine is not called in a way await/await
-        loop.set_debug(True)
-
-    await Controller.instance().start()
-    # Because with a large image collection
-    # without md5sum already computed we start the
-    # computing with server start
-
-    from gns3server.compute.qemu import Qemu
-    asyncio.ensure_future(Qemu.instance().list_images())
-
-    for module in MODULES:
-        log.debug("Loading module {}".format(module.__name__))
-        m = module.instance()
-        m.port_manager = PortManager.instance()
-
-
-@app.on_event("shutdown")
-async def shutdown_event():
-
-    await HTTPClient.close_session()
-    await Controller.instance().stop()
-
-    for module in MODULES:
-        log.debug("Unloading module {}".format(module.__name__))
-        m = module.instance()
-        await m.unload()
-
-    if PortManager.instance().tcp_ports:
-        log.warning("TCP ports are still used {}".format(PortManager.instance().tcp_ports))
-
-    if PortManager.instance().udp_ports:
-        log.warning("UDP ports are still used {}".format(PortManager.instance().udp_ports))

gns3server/compute/base_node.py

@@ -31,8 +31,8 @@ from gns3server.compute.compute_error import ComputeError
 from ..compute.port_manager import PortManager
 from ..utils.asyncio import wait_run_in_executor, locking
 from ..utils.asyncio.telnet_server import AsyncioTelnetServer
-from ..ubridge.hypervisor import Hypervisor
+from gns3server.compute.ubridge.hypervisor import Hypervisor
-from ..ubridge.ubridge_error import UbridgeError
+from gns3server.compute.ubridge.ubridge_error import UbridgeError
 from .nios.nio_udp import NIOUDP
 from .error import NodeError
 

gns3server/compute/builtin/nodes/cloud.py

@@ -21,7 +21,7 @@ import subprocess
 from ...error import NodeError
 from ...base_node import BaseNode
 from ...nios.nio_udp import NIOUDP
-from ....ubridge.ubridge_error import UbridgeError
+from gns3server.compute.ubridge.ubridge_error import UbridgeError
 
 import gns3server.utils.interfaces
 import gns3server.utils.asyncio

gns3server/compute/docker/docker_vm.py

@@ -35,7 +35,7 @@ from gns3server.utils.asyncio import wait_for_file_creation
 from gns3server.utils.asyncio import monitor_process
 from gns3server.utils.get_resource import get_resource
 
-from gns3server.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError
+from gns3server.compute.ubridge.ubridge_error import UbridgeError, UbridgeNamespaceError
 from ..base_node import BaseNode
 
 from ..adapters.ethernet_adapter import EthernetAdapter

gns3server/compute/iou/iou_vm.py

@@ -40,7 +40,7 @@ from ..nios.nio_udp import NIOUDP
 from ..base_node import BaseNode
 from .utils.iou_import import nvram_import
 from .utils.iou_export import nvram_export
-from gns3server.ubridge.ubridge_error import UbridgeError
+from gns3server.compute.ubridge.ubridge_error import UbridgeError
 from gns3server.utils.file_watcher import FileWatcher
 from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer
 from gns3server.utils.asyncio import locking

gns3server/compute/notification_manager.py

@@ -17,7 +17,7 @@
 
 
 from contextlib import contextmanager
-from ..notification_queue import NotificationQueue
+from gns3server.utils.notification_queue import NotificationQueue
 
 
 class NotificationManager:

gns3server/compute/ubridge/ubridge_hypervisor.py

@@ -20,7 +20,7 @@ import time
 import logging
 import asyncio
 
-from ..utils.asyncio import locking
+from gns3server.utils.asyncio import locking
 from .ubridge_error import UbridgeError
 
 log = logging.getLogger(__name__)

gns3server/controller/notification.py

@@ -18,7 +18,7 @@
 import os
 from contextlib import contextmanager
 
-from ..notification_queue import NotificationQueue
+from gns3server.utils.notification_queue import NotificationQueue
 from .controller_error import ControllerError
 
 

gns3server/core/tasks.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2020 GNS3 Technologies Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import sys
+import asyncio
+
+from typing import Callable
+from fastapi import FastAPI
+
+from gns3server.controller import Controller
+from gns3server.compute import MODULES
+from gns3server.compute.port_manager import PortManager
+from gns3server.utils.http_client import HTTPClient
+#from gns3server.db.tasks import connect_to_db, close_db_connection
+
+import logging
+log = logging.getLogger(__name__)
+
+
+def create_startup_handler(app: FastAPI) -> Callable:
+
+    async def start_app() -> None:
+        loop = asyncio.get_event_loop()
+        logger = logging.getLogger("asyncio")
+        logger.setLevel(logging.ERROR)
+
+        if sys.platform.startswith("win"):
+            # Add a periodic callback to give a chance to process signals on Windows
+            # because asyncio.add_signal_handler() is not supported yet on that platform
+            # otherwise the loop runs outside of signal module's ability to trap signals.
+
+            def wakeup():
+                loop.call_later(0.5, wakeup)
+
+            loop.call_later(0.5, wakeup)
+
+        if log.getEffectiveLevel() == logging.DEBUG:
+            # On debug version we enable info that
+            # coroutine is not called in a way await/await
+            loop.set_debug(True)
+
+        # connect to the database
+        # await connect_to_db(app)
+
+        await Controller.instance().start()
+        # Because with a large image collection
+        # without md5sum already computed we start the
+        # computing with server start
+
+        from gns3server.compute.qemu import Qemu
+        asyncio.ensure_future(Qemu.instance().list_images())
+
+        for module in MODULES:
+            log.debug("Loading module {}".format(module.__name__))
+            m = module.instance()
+            m.port_manager = PortManager.instance()
+
+    return start_app
+
+
+def create_shutdown_handler(app: FastAPI) -> Callable:
+
+    async def shutdown_handler() -> None:
+        await HTTPClient.close_session()
+        await Controller.instance().stop()
+
+        for module in MODULES:
+            log.debug("Unloading module {}".format(module.__name__))
+            m = module.instance()
+            await m.unload()
+
+        if PortManager.instance().tcp_ports:
+            log.warning("TCP ports are still used {}".format(PortManager.instance().tcp_ports))
+
+        if PortManager.instance().udp_ports:
+            log.warning("UDP ports are still used {}".format(PortManager.instance().udp_ports))
+
+        # close the connection to the database
+        # await close_db_connection(app)
+
+    return shutdown_handler

gns3server/run.py

@@ -38,6 +38,7 @@ from gns3server.logger import init_logger
 from gns3server.version import __version__
 from gns3server.config import Config
 from gns3server.crash_report import CrashReport
+from gns3server.api.server import app
 
 
 import logging
@@ -326,7 +327,7 @@ def run():
             certkey = server_config["certkey"]
             log.info("SSL is enabled")
 
-        config = uvicorn.Config("gns3server.app:app",
+        config = uvicorn.Config(app,
                                 host=host,
                                 port=port,
                                 access_log=access_log,

scripts/publish_api_documentation.py

@@ -20,7 +20,7 @@
 import json
 
 from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
-from gns3server.app import app
+from gns3server.api.server import app
 
 
 if __name__ == "__main__":

tests/api/routes/test_index.py

@@ -28,7 +28,7 @@ from gns3server.utils.get_resource import get_resource
 def get_static(filename):
 
     current_dir = os.path.dirname(os.path.abspath(__file__))
-    return os.path.join(os.path.abspath(os.path.join(current_dir, '..', '..', 'gns3server', 'static')), filename)
+    return os.path.join(os.path.abspath(os.path.join(current_dir, '../..', '..', 'gns3server', 'static')), filename)
 
 
 @pytest.mark.asyncio

tests/compute/docker/test_docker_vm.py

@@ -23,9 +23,9 @@ import sys
 import os
 from tests.utils import asyncio_patch, AsyncioMagicMock
 
-from gns3server.ubridge.ubridge_error import UbridgeNamespaceError
+from gns3server.compute.ubridge.ubridge_error import UbridgeNamespaceError
 from gns3server.compute.docker.docker_vm import DockerVM
-from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error, DockerHttp304Error
+from gns3server.compute.docker.docker_error import DockerError, DockerHttp404Error
 from gns3server.compute.docker import Docker
 from gns3server.utils.get_resource import get_resource
 

tests/conftest.py

@@ -15,13 +15,13 @@ from gns3server.compute.port_manager import PortManager
 from gns3server.compute.project_manager import ProjectManager
 
 
-from .endpoints.base import Query
+from tests.api.routes.base import Query
 
 sys._called_from_test = True
 sys.original_platform = sys.platform
 
 from fastapi.testclient import TestClient
-from gns3server.app import app
+from gns3server.api.server import app
 from httpx import AsyncClient