Close project if one one the compute of the project is down

Fix #836

gns3server/controller/__init__.py

@@ -270,6 +270,15 @@ class Controller:
             self.notification.emit("compute.updated", self._computes[compute_id].__json__())
             return self._computes[compute_id]
 
+    @asyncio.coroutine
+    def close_compute_projects(self, compute):
+        """
+        Close projects running on a compute
+        """
+        for project in self._projects.values():
+            if compute in project.computes:
+                yield from project.close()
+
     @asyncio.coroutine
     def delete_compute(self, compute_id):
         """
@@ -281,11 +290,7 @@ class Controller:
             compute = self.get_compute(compute_id)
         except aiohttp.web.HTTPNotFound:
             return
-
-        for project in self._projects.values():
-            if compute in project.computes:
-                yield from project.close()
-
+        yield from self.close_compute_projects(compute)
         yield from compute.close()
         del self._computes[compute_id]
         self.save()

gns3server/controller/compute.py

@@ -107,6 +107,8 @@ class Compute:
         # Cache of interfaces on remote host
         self._interfaces_cache = None
 
+        self._connection_failure = 0
+
     def _session(self):
         if self._http_session is None or self._http_session.closed is True:
             self._http_session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(limit=None, force_close=True))
@@ -344,14 +346,17 @@ class Compute:
         return StreamResponse(response)
 
     @asyncio.coroutine
-    def http_query(self, method, path, data=None, **kwargs):
-        if not self._connected:
+    def http_query(self, method, path, data=None, dont_connect=False, **kwargs):
+        """
+        :param dont_connect: If true do not reconnect if not connected
+        """
+        if not self._connected and not dont_connect:
             if self._id == "vm" and not self._controller.gns3vm.running:
                 yield from self._controller.gns3vm.start()
 
             yield from self.connect()
-        if not self._connected:
-            raise aiohttp.web.HTTPConflict(text="Can't connect to {}".format(self._name))
+        if not self._connected and not dont_connect:
+            raise ComputeError("Can't connect to {}".format(self._name))
         response = yield from self._run_http_query(method, path, data=data, **kwargs)
         return response
 
@@ -366,7 +371,12 @@ class Compute:
             except ComputeError:
                 # Try to reconnect after 2 seconds if server unavailable only if not during tests (otherwise we create a ressources usage bomb)
                 if not hasattr(sys, "_called_from_test") or not sys._called_from_test:
+                    self._connection_failure += 1
+                    # After 5 failure we close the project using the compute to avoid sync issues
+                    if self._connection_failure == 5:
+                        yield from self._controller.close_compute_projects(self)
                     asyncio.get_event_loop().call_later(2, lambda: asyncio.async(self.connect()))
+
                 return
             except aiohttp.web.HTTPNotFound:
                 raise aiohttp.web.HTTPConflict(text="The server {} is not a GNS3 server or it's a 1.X server".format(self._id))
@@ -383,6 +393,7 @@ class Compute:
 
             self._notifications = asyncio.gather(self._connect_notification())
             self._connected = True
+            self._connection_failure = 0
             self._controller.notification.emit("compute.updated", self.__json__())
 
     @asyncio.coroutine

gns3server/controller/node.py

@@ -425,7 +425,7 @@ class Node:
         Stop a node
         """
         try:
-            yield from self.post("/stop", timeout=240)
+            yield from self.post("/stop", timeout=240, dont_connect=True)
         # We don't care if a node is down at this step
         except (ComputeError, aiohttp.errors.ClientHttpProcessingError, aiohttp.web.HTTPError):
             pass

gns3server/controller/project.py

@@ -541,7 +541,7 @@ class Project:
         yield from self.stop_all()
         for compute in self._project_created_on_compute:
             try:
-                yield from compute.post("/projects/{}/close".format(self._id))
+                yield from compute.post("/projects/{}/close".format(self._id), dont_connect=True)
             # We don't care if a compute is down at this step
             except (ComputeError, aiohttp.web.HTTPError, aiohttp.ClientResponseError, TimeoutError):
                 pass

gns3server/utils/asyncio/pool.py

@@ -36,6 +36,7 @@ class Pool():
         Wait for all task to finish
         """
         pending = set()
+        exceptions = set()
         while len(self._tasks) > 0 or len(pending) > 0:
             while len(self._tasks) > 0 and len(pending) < self._concurrency:
                 task, args, kwargs = self._tasks.pop(0)
@@ -43,7 +44,9 @@ class Pool():
             (done, pending) = yield from asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
             for task in done:
                 if task.exception():
-                    raise task.exception()
+                    exceptions.add(task.exception())
+        if len(exceptions) > 0:
+            raise exceptions.pop()
 
 
 def main():

tests/controller/test_node.py

@@ -366,7 +366,7 @@ def test_stop(node, compute, project, async_run):
     compute.post = AsyncioMagicMock()
 
     async_run(node.stop())
-    compute.post.assert_called_with("/projects/{}/vpcs/nodes/{}/stop".format(node.project.id, node.id), timeout=240)
+    compute.post.assert_called_with("/projects/{}/vpcs/nodes/{}/stop".format(node.project.id, node.id), timeout=240, dont_connect=True)
 
 
 def test_suspend(node, compute, project, async_run):