Fix installation with Python 3.7. Fixes #1414.

Fix deprecated use of aiohttp.Timeout. Fixes #1296.
Use "async with" with aiohttp.ClientSession().
Make sure websocket connections are properly closed, see https://docs.aiohttp.org/en/stable/web_advanced.html#graceful-shutdown
Finish to drop Python 3.4.

gns3server/compute/docker/__init__.py

@@ -93,6 +93,8 @@ class Docker(BaseManager):
         if self._connected:
             if self._connector and not self._connector.closed:
                 self._connector.close()
+        if self._session and not self._session.closed:
+            await self._session.close()
 
     async def query(self, method, path, data={}, params={}):
         """
@@ -106,6 +108,7 @@ class Docker(BaseManager):
 
         response = await self.http_query(method, path, data=data, params=params)
         body = await response.read()
+        response.close()
         if body and len(body):
             if response.headers['CONTENT-TYPE'] == 'application/json':
                 body = json.loads(body.decode("utf-8"))
@@ -140,14 +143,12 @@ class Docker(BaseManager):
             if self._session is None or self._session.closed:
                 connector = self.connector()
                 self._session = aiohttp.ClientSession(connector=connector)
-            response = await self._session.request(
+            response = await self._session.request(method,
-                method,
                                                    url,
                                                    params=params,
                                                    data=data,
                                                    headers={"content-type": "application/json", },
-                timeout=timeout
+                                                   timeout=timeout)
-            )
         except (aiohttp.ClientResponseError, aiohttp.ClientOSError) as e:
             raise DockerError("Docker has returned an error: {}".format(str(e)))
         except (asyncio.TimeoutError):
@@ -177,9 +178,7 @@ class Docker(BaseManager):
         """
 
         url = "http://docker/v" + self._api_version + "/" + path
-        connection = await self._session.ws_connect(url,
+        connection = await self._session.ws_connect(url, origin="http://docker", autoping=True)
-                                                         origin="http://docker",
-                                                         autoping=True)
         return connection
 
     @locking

gns3server/controller/__init__.py

@@ -67,13 +67,13 @@ class Controller:
     @locking
     async def download_appliance_templates(self):
 
-        session = aiohttp.ClientSession()
         try:
             headers = {}
             if self._appliance_templates_etag:
                 log.info("Checking if appliance templates are up-to-date (ETag {})".format(self._appliance_templates_etag))
                 headers["If-None-Match"] = self._appliance_templates_etag
-            response = await session.get('https://api.github.com/repos/GNS3/gns3-registry/contents/appliances', headers=headers)
+            async with aiohttp.ClientSession() as session:
+                async with session.get('https://api.github.com/repos/GNS3/gns3-registry/contents/appliances', headers=headers) as response:
                     if response.status == 304:
                         log.info("Appliance templates are already up-to-date (ETag {})".format(self._appliance_templates_etag))
                         return
@@ -84,13 +84,12 @@ class Controller:
                         self._appliance_templates_etag = etag
                         self.save()
                     json_data = await response.json()
-            response.close()
                 appliances_dir = get_resource('appliances')
                 for appliance in json_data:
                     if appliance["type"] == "file":
                         appliance_name = appliance["name"]
                         log.info("Download appliance template file from '{}'".format(appliance["download_url"]))
-                    response = await session.get(appliance["download_url"])
+                        async with session.get(appliance["download_url"]) as response:
                             if response.status != 200:
                                 log.warning("Could not download '{}' due to HTTP error code {}".format(appliance["download_url"], response.status))
                                 continue
@@ -100,7 +99,6 @@ class Controller:
                                 log.warning("Timeout while downloading '{}'".format(appliance["download_url"]))
                                 continue
                             path = os.path.join(appliances_dir, appliance_name)
-
                             try:
                                 log.info("Saving {} file to {}".format(appliance_name, path))
                                 with open(path, 'wb') as f:
@@ -109,8 +107,6 @@ class Controller:
                                 raise aiohttp.web.HTTPConflict(text="Could not write appliance template file '{}': {}".format(path, e))
         except ValueError as e:
             raise aiohttp.web.HTTPConflict(text="Could not read appliance templates information from GitHub: {}".format(e))
-        finally:
-            session.close()
 
     def load_appliance_templates(self):
 

gns3server/controller/compute.py

@@ -18,6 +18,7 @@
 import ipaddress
 import aiohttp
 import asyncio
+import async_timeout
 import socket
 import json
 import uuid
@@ -52,22 +53,6 @@ class ComputeConflict(aiohttp.web.HTTPConflict):
         self.response = response
 
 
-class Timeout(aiohttp.Timeout):
-    """
-    Could be removed with aiohttp 0.22 that support None timeout
-    """
-
-    def __enter__(self):
-        if self._timeout:
-            return super().__enter__()
-        return self
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self._timeout:
-            return super().__exit__(exc_type, exc_val, exc_tb)
-        return self
-
-
 class Compute:
     """
     A GNS3 compute.
@@ -101,12 +86,8 @@ class Compute:
             "node_types": []
         }
         self.name = name
-        # Websocket for notifications
-        self._ws = None
-
         # Cache of interfaces on remote host
         self._interfaces_cache = None
-
         self._connection_failure = 0
 
     def _session(self):
@@ -114,9 +95,10 @@ class Compute:
             self._http_session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=None, force_close=True))
         return self._http_session
 
-    def __del__(self):
+    #def __del__(self):
-        if self._http_session:
+    #    pass
-            self._http_session.close()
+    #    if self._http_session:
+    #        self._http_session.close()
 
     def _set_auth(self, user, password):
         """
@@ -162,19 +144,16 @@ class Compute:
         # It's important to set user and password at the same time
         if "user" in kwargs or "password" in kwargs:
             self._set_auth(kwargs.get("user", self._user), kwargs.get("password", self._password))
-        if self._http_session:
+        if self._http_session and not self._http_session.closed:
-            self._http_session.close()
+            await self._http_session.close()
         self._connected = False
         self._controller.notification.controller_emit("compute.updated", self.__json__())
         self._controller.save()
 
     async def close(self):
         self._connected = False
-        if self._http_session:
+        if self._http_session and not self._http_session.closed:
-            self._http_session.close()
+            await self._http_session.close()
-        if self._ws:
-            await self._ws.close()
-            self._ws = None
         self._closed = True
 
     @property
@@ -474,19 +453,10 @@ class Compute:
         """
         Connect to the notification stream
         """
-        try:
+
-            self._ws = await self._session().ws_connect(self._getUrl("/notifications/ws"), auth=self._auth)
+        async with self._session().ws_connect(self._getUrl("/notifications/ws"), auth=self._auth) as ws:
-        except (aiohttp.WSServerHandshakeError, aiohttp.ClientResponseError):
+            async for response in ws:
-            self._ws = None
+                if response.type == aiohttp.WSMsgType.TEXT and response.data:
-        while self._ws is not None:
-            try:
-                response = await self._ws.receive()
-            except aiohttp.WSServerHandshakeError:
-                self._ws = None
-                break
-            if response.tp == aiohttp.WSMsgType.closed or response.tp == aiohttp.WSMsgType.error or response.data is None:
-                self._connected = False
-                break
                     msg = json.loads(response.data)
                     action = msg.pop("action")
                     event = msg.pop("event")
@@ -496,13 +466,14 @@ class Compute:
                         self._controller.notification.controller_emit("compute.updated", self.__json__())
                     else:
                         await self._controller.notification.dispatch(action, event, compute_id=self.id)
-        if self._ws:
+                elif response.type == aiohttp.WSMsgType.CLOSED or response.type == aiohttp.WSMsgType.ERROR or response.data is None:
-            await self._ws.close()
+                    self._connected = False
+                    break
 
         # Try to reconnect after 1 seconds if server unavailable only if not during tests (otherwise we create a ressources usage bomb)
         if not hasattr(sys, "_called_from_test") or not sys._called_from_test:
             asyncio.get_event_loop().call_later(1, lambda: asyncio.ensure_future(self.connect()))
-        self._ws = None
+
         self._cpu_usage_percent = None
         self._memory_usage_percent = None
         self._controller.notification.controller_emit("compute.updated", self.__json__())
@@ -527,7 +498,7 @@ class Compute:
         return self._getUrl(path)
 
     async def _run_http_query(self, method, path, data=None, timeout=20, raw=False):
-        with Timeout(timeout):
+        with async_timeout.timeout(timeout):
             url = self._getUrl(path)
             headers = {}
             headers['content-type'] = 'application/json'

gns3server/controller/gns3vm/virtualbox_gns3_vm.py

@@ -261,29 +261,22 @@ class VirtualBoxGNS3VM(BaseGNS3VM):
         """
         remaining_try = 300
         while remaining_try > 0:
-            json_data = None
+            async with aiohttp.ClientSession() as session:
-            session = aiohttp.ClientSession()
                 try:
-                resp = None
+                    async with session.get('http://127.0.0.1:{}/v2/compute/network/interfaces'.format(api_port)) as resp:
-                resp = await session.get('http://127.0.0.1:{}/v2/compute/network/interfaces'.format(api_port))
-            except (OSError, aiohttp.ClientError, TimeoutError, asyncio.TimeoutError):
-                pass
-
-            if resp:
                         if resp.status < 300:
                             try:
                                 json_data = await resp.json()
-                    except ValueError:
-                        pass
-                resp.close()
-
-            session.close()
-
                                 if json_data:
                                     for interface in json_data:
-                    if "name" in interface and interface["name"] == "eth{}".format(hostonly_interface_number - 1):
+                                        if "name" in interface and interface["name"] == "eth{}".format(
+                                                hostonly_interface_number - 1):
                                             if "ip_address" in interface and len(interface["ip_address"]) > 0:
                                                 return interface["ip_address"]
+                            except ValueError:
+                                pass
+                except (OSError, aiohttp.ClientError, TimeoutError, asyncio.TimeoutError):
+                    pass
             remaining_try -= 1
             await asyncio.sleep(1)
         raise GNS3VMError("Could not get the GNS3 VM ip make sure the VM receive an IP from VirtualBox")

gns3server/handlers/api/compute/notification_handler.py

@@ -42,15 +42,17 @@ class NotificationHandler:
         ws = WebSocketResponse()
         await ws.prepare(request)
 
+        request.app['websockets'].add(ws)
         asyncio.ensure_future(process_websocket(ws))
-
         with notifications.queue() as queue:
             while True:
                 try:
                     notification = await queue.get_json(1)
-                except asyncio.futures.CancelledError:
-                    break
                     if ws.closed:
                         break
-                ws.send_str(notification)
+                    await ws.send_str(notification)
+                except asyncio.futures.CancelledError:
+                    break
+                finally:
+                    request.app['websockets'].discard(ws)
         return ws

gns3server/handlers/api/controller/notification_handler.py

@@ -69,14 +69,17 @@ class NotificationHandler:
         ws = aiohttp.web.WebSocketResponse()
         await ws.prepare(request)
 
+        request.app['websockets'].add(ws)
         asyncio.ensure_future(process_websocket(ws))
         with controller.notification.controller_queue() as queue:
             while True:
                 try:
                     notification = await queue.get_json(5)
-                except asyncio.futures.CancelledError:
-                    break
                     if ws.closed:
                         break
-                ws.send_str(notification)
+                    await ws.send_str(notification)
+                except asyncio.futures.CancelledError:
+                    break
+                finally:
+                    request.app['websockets'].discard(ws)
         return ws

gns3server/handlers/api/controller/project_handler.py

@@ -261,17 +261,19 @@ class ProjectHandler:
         ws = aiohttp.web.WebSocketResponse()
         await ws.prepare(request)
 
+        request.app['websockets'].add(ws)
         asyncio.ensure_future(process_websocket(ws))
-
         with controller.notification.project_queue(project) as queue:
             while True:
                 try:
                     notification = await queue.get_json(5)
-                except asyncio.futures.CancelledError as e:
-                    break
                     if ws.closed:
                         break
-                ws.send_str(notification)
+                    await ws.send_str(notification)
+                except asyncio.futures.CancelledError:
+                    break
+                finally:
+                    request.app['websockets'].discard(ws)
 
         if project.auto_close:
             # To avoid trouble with client connecting disconnecting we sleep few seconds before checking

gns3server/web/response.py

@@ -51,6 +51,7 @@ class Response(aiohttp.web.Response):
         super().enable_chunked_encoding()
 
     async def prepare(self, request):
+
         if log.getEffectiveLevel() == logging.DEBUG:
             log.info("%s %s", request.method, request.path_qs)
             log.debug("%s", dict(request.headers))

gns3server/web/web_server.py

@@ -28,6 +28,7 @@ import aiohttp_cors
 import functools
 import time
 import atexit
+import weakref
 
 # Import encoding now, to avoid implicit import later.
 # Implicit import within threads may cause LookupError when standard library is in a ZIP
@@ -48,8 +49,8 @@ import gns3server.handlers
 import logging
 log = logging.getLogger(__name__)
 
-if not (aiohttp.__version__.startswith("2.2") or aiohttp.__version__.startswith("2.3")):
+if not (aiohttp.__version__.startswith("3.")):
-    raise RuntimeError("aiohttp 2.2.x or 2.3.x is required to run the GNS3 server")
+    raise RuntimeError("aiohttp 3.x is required to run the GNS3 server")
 
 
 class WebServer:
@@ -100,17 +101,16 @@ class WebServer:
             log.warning("Close is already in progress")
             return
 
+        # close websocket connections
+        for ws in set(self._app['websockets']):
+            await ws.close(code=aiohttp.WSCloseCode.GOING_AWAY, message='Server shutdown')
+
         if self._server:
             self._server.close()
             await self._server.wait_closed()
         if self._app:
             await self._app.shutdown()
         if self._handler:
-            try:
-                # aiohttp < 2.3
-                await self._handler.finish_connections(2)  # Parameter is timeout
-            except AttributeError:
-                # aiohttp >= 2.3
             await self._handler.shutdown(2)  # Parameter is timeout
         if self._app:
             await self._app.cleanup()
@@ -254,6 +254,10 @@ class WebServer:
             log.debug("ENV %s=%s", key, val)
 
         self._app = aiohttp.web.Application()
+
+        # Keep a list of active websocket connections
+        self._app['websockets'] = weakref.WeakSet()
+
         # Background task started with the server
         self._app.on_startup.append(self._on_startup)
 

requirements.txt

@@ -1,12 +1,10 @@
 jsonschema>=2.4.0
-aiohttp>=2.3.3,<2.4.0 # pyup: ignore
+aiohttp==3.2.1
-aiohttp-cors>=0.5.3,<0.6.0 # pyup: ignore
+aiohttp-cors==0.7.0
-yarl>=0.11
 Jinja2>=2.7.3
 raven>=5.23.0
 psutil>=3.0.0
 zipstream>=1.1.4
-typing>=3.5.3.0 # Otherwise yarl fails with python 3.4
 prompt-toolkit==1.0.15
-async-timeout<3.0.0 # pyup: ignore; 3.0 drops support for python 3.4
+async-timeout==3.0.1
 distro>=1.3.0

setup.py

@@ -19,9 +19,9 @@ import sys
 from setuptools import setup, find_packages
 from setuptools.command.test import test as TestCommand
 
-# we only support Python 3 version >= 3.4
+# we only support Python 3 version >= 3.5.3
-if len(sys.argv) >= 2 and sys.argv[1] == "install" and sys.version_info < (3, 4):
+if len(sys.argv) >= 2 and sys.argv[1] == "install" and sys.version_info < (3, 5, 3):
-    raise SystemExit("Python 3.4 or higher is required")
+    raise SystemExit("Python 3.5.3 or higher is required")
 
 
 class PyTest(TestCommand):