From edae834e0e68137ac7db6913cef73960ba9f7498 Mon Sep 17 00:00:00 2001
From: Julien Duponchelle <julien@gns3.net>
Date: Mon, 29 Aug 2016 17:36:24 +0200
Subject: [PATCH] Monitor status of the compute

---
 gns3server/controller/compute.py | 21 ++++++++++++++++-----
 tests/controller/test_compute.py |  5 ++---
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py
index 2ef1ff6d..33dac0ab 100644
--- a/gns3server/controller/compute.py
+++ b/gns3server/controller/compute.py
@@ -88,6 +88,7 @@ class Compute:
         self._user = None
         self._password = None
         self._connected = False
+        self._closed = False  # Close mean we are destroying the compute node
         self._controller = controller
         self._set_auth(user, password)
         self._cpu_usage_percent = None
@@ -157,6 +158,7 @@ class Compute:
         if self._ws:
             yield from self._ws.close()
             self._ws = None
+        self._closed = True
 
     @property
     def name(self):
@@ -333,7 +335,7 @@ class Compute:
         if not self._connected:
             yield from self.connect()
         if not self._connected:
-            raise aiohttp.web.HTTPConflict(text="The server {} is not a GNS3 server".format(self._id))
+            raise aiohttp.web.HTTPConflict(text="Can't connect to {}".format(self._name))
         response = yield from self._run_http_query(method, path, data=data, **kwargs)
         return response
 
@@ -342,10 +344,12 @@ class Compute:
         """
         Check if remote server is accessible
         """
-        if not self._connected:
+        if not self._connected and not self._closed:
             try:
                 response = yield from self._run_http_query("GET", "/capabilities")
-            except aiohttp.errors.ClientOSError:
+            except (aiohttp.errors.ClientOSError, aiohttp.errors.ClientRequestError):
+                # Try to reconnect after 2 seconds if server unavailable
+                asyncio.get_event_loop().call_later(2, lambda: asyncio.async(self.connect()))
                 return
 
             if "version" not in response.json:
@@ -371,7 +375,7 @@ class Compute:
                 response = yield from self._ws.receive()
             except aiohttp.errors.WSServerHandshakeError:
                 self._ws = None
-                return
+                break
             if response.tp == aiohttp.MsgType.closed or response.tp == aiohttp.MsgType.error:
                 self._connected = False
                 break
@@ -385,8 +389,15 @@ class Compute:
                 self._controller.notification.emit("compute.updated", self.__json__())
             else:
                 self._controller.notification.dispatch(action, event, compute_id=self.id)
-        yield from self._ws.close()
+        if self._ws:
+            yield from self._ws.close()
+
+        # Try to reconnect after 1 seconds if server unavailable
+        asyncio.get_event_loop().call_later(1, lambda: asyncio.async(self.connect()))
         self._ws = None
+        self._cpu_usage_percent = None
+        self._memory_usage_percent = None
+        self._controller.notification.emit("compute.updated", self.__json__())
 
     def _getUrl(self, path):
         return "{}://{}:{}/v2/compute{}".format(self._protocol, self._host, self._port, path)
diff --git a/tests/controller/test_compute.py b/tests/controller/test_compute.py
index 349cd77d..7177150e 100644
--- a/tests/controller/test_compute.py
+++ b/tests/controller/test_compute.py
@@ -213,11 +213,10 @@ def test_connectNotificationPing(compute, async_run):
     async_run(compute._connect_notification())
 
     assert not compute._controller.notification.dispatch.called
-    assert compute.cpu_usage_percent == 35.7
-    assert compute.memory_usage_percent == 80.7
-    args, _ = compute._controller.notification.emit.call_args
+    args, _ = compute._controller.notification.emit.call_args_list[0]
     assert args[0] == "compute.updated"
     assert args[1]["memory_usage_percent"] == 80.7
+    assert args[1]["cpu_usage_percent"] == 35.7
 
 
 def test_json(compute):