Auto upload missing images

gns3server/compute/base_manager.py

@@ -38,7 +38,7 @@ from .project_manager import ProjectManager
 from .nios.nio_udp import NIOUDP
 from .nios.nio_tap import NIOTAP
 from .nios.nio_ethernet import NIOEthernet
-from ..utils.images import md5sum, remove_checksum
+from ..utils.images import md5sum, remove_checksum, images_directories
 from .error import NodeError, ImageMissingError
 
 
@@ -389,24 +389,6 @@ class BaseManager:
         assert nio is not None
         return nio
 
-    def images_directories(self):
-        """
-        Return all directory where we will look for images
-        by priority
-        """
-        server_config = self.config.get_section_config("Server")
-
-        paths = []
-        img_directory = self.get_images_directory()
-        os.makedirs(img_directory, exist_ok=True)
-        paths.append(img_directory)
-        for directory in server_config.get("additional_images_path", "").split(":"):
-            paths.append(directory)
-        # Compatibility with old topologies we look in parent directory
-        paths.append(os.path.normpath(os.path.join(self.get_images_directory(), '..')))
-        # Return only the existings paths
-        return [force_unix_path(p) for p in paths if os.path.exists(p)]
-
     def get_abs_image_path(self, path):
         """
         Get the absolute path of an image
@@ -417,6 +399,7 @@ class BaseManager:
 
         if not path:
             return ""
+        orig_path = path
 
         server_config = self.config.get_section_config("Server")
         img_directory = self.get_images_directory()
@@ -427,8 +410,7 @@ class BaseManager:
                 raise NodeError("{} is not allowed on this remote server. Please use only a filename in {}.".format(path, img_directory))
 
         if not os.path.isabs(path):
-            orig_path = path
+            for directory in images_directories(self._NODE_TYPE):
-            for directory in self.images_directories():
                 path = self._recursive_search_file_in_directory(directory, orig_path)
                 if path:
                     return force_unix_path(path)
@@ -438,21 +420,21 @@ class BaseManager:
             path = force_unix_path(os.path.join(self.get_images_directory(), *s))
             if os.path.exists(path):
                 return path
-            raise ImageMissingError(path)
+            raise ImageMissingError(orig_path)
 
         # For non local server we disallow using absolute path outside image directory
         if server_config.get("local", False) is True:
             path = force_unix_path(path)
             if os.path.exists(path):
                 return path
-            raise ImageMissingError(path)
+            raise ImageMissingError(orig_path)
 
         path = force_unix_path(path)
-        for directory in self.images_directories():
+        for directory in images_directories(self._NODE_TYPE):
             if os.path.commonprefix([directory, path]) == directory:
                 if os.path.exists(path):
                     return path
-                raise ImageMissingError(path)
+                raise ImageMissingError(orig_path)
         raise NodeError("{} is not allowed on this remote server. Please use only a filename in {}.".format(path, self.get_images_directory()))
 
     def _recursive_search_file_in_directory(self, directory, searched_file):
@@ -485,7 +467,7 @@ class BaseManager:
         if not path:
             return ""
         path = force_unix_path(self.get_abs_image_path(path))
-        for directory in self.images_directories():
+        for directory in images_directories(self._NODE_TYPE):
             if os.path.commonprefix([directory, path]) == directory:
                 return os.path.relpath(path, directory)
         return path

gns3server/compute/dynamips/__init__.py

@@ -103,6 +103,7 @@ WIC_MATRIX = {"WIC-1ENET": WIC_1ENET,
 class Dynamips(BaseManager):
 
     _NODE_CLASS = DynamipsVMFactory
+    _NODE_TYPE = "dynamips"
     _DEVICE_CLASS = DynamipsDeviceFactory
     _ghost_ios_lock = None
 

gns3server/compute/error.py

@@ -39,3 +39,4 @@ class ImageMissingError(Exception):
 
     def __init__(self, image):
         super().__init__("The image {} is missing".format(image))
+        self.image = image

gns3server/compute/iou/__init__.py

@@ -33,6 +33,7 @@ log = logging.getLogger(__name__)
 class IOU(BaseManager):
 
     _NODE_CLASS = IOUVM
+    _NODE_TYPE = "iou"
 
     def __init__(self):
 

gns3server/compute/qemu/__init__.py

@@ -38,6 +38,7 @@ log = logging.getLogger(__name__)
 class Qemu(BaseManager):
 
     _NODE_CLASS = QemuVM
+    _NODE_TYPE = "qemu"
 
     @staticmethod
     @asyncio.coroutine

gns3server/controller/compute.py

@@ -19,6 +19,7 @@ import aiohttp
 import asyncio
 import json
 import uuid
+import io
 
 from ..utils import parse_version
 from ..controller.controller_error import ControllerError
@@ -34,6 +35,17 @@ class ComputeError(ControllerError):
     pass
 
 
+class ComputeConflict(aiohttp.web.HTTPConflict):
+    """
+    Raise when the compute send a 409 that we can handle
+
+    :param response: The response of the compute
+    """
+    def __init__(self, response):
+        super().__init__(text=response["message"])
+        self.response = response
+
+
 class Timeout(aiohttp.Timeout):
     """
     Could be removed with aiohttp 0.22 that support None timeout
@@ -293,7 +305,7 @@ class Compute:
                 if hasattr(data, '__json__'):
                     data = json.dumps(data.__json__())
                 # Stream the request
-                elif isinstance(data, aiohttp.streams.StreamReader):
+                elif isinstance(data, aiohttp.streams.StreamReader) or isinstance(data, io.BufferedIOBase):
                     chunked = True
                     headers['content-type'] = 'application/octet-stream'
                 else:
@@ -306,10 +318,13 @@ class Compute:
 
             if response.status >= 300:
                 # Try to decode the GNS3 error
+                if body:
                     try:
                         msg = json.loads(body)["message"]
                     except (KeyError, json.decoder.JSONDecodeError):
                         msg = body
+                else:
+                    msg = ""
 
                 if response.status == 400:
                     raise aiohttp.web.HTTPBadRequest(text="Bad request {} {}".format(url, body))
@@ -320,6 +335,10 @@ class Compute:
                 elif response.status == 404:
                     raise aiohttp.web.HTTPNotFound(text=msg)
                 elif response.status == 409:
+                    try:
+                        raise ComputeConflict(json.loads(body))
+                    # If the 409 doesn't come from a GNS3 server
+                    except json.decoder.JSONDecodeError:
                         raise aiohttp.web.HTTPConflict(text=msg)
                 elif response.status == 500:
                     raise aiohttp.web.HTTPInternalServerError(text="Internal server error {}".format(url))

gns3server/controller/node.py

@@ -15,12 +15,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-
+import aiohttp
 import asyncio
 import copy
 import uuid
+import os
 
 
+from .compute import ComputeConflict
+from ..utils.images import images_directories
+
 class Node:
 
     def __init__(self, project, compute, node_id=None, node_type=None, name=None, console=None, console_type=None, properties={}):
@@ -101,8 +105,19 @@ class Node:
         """
         data = self._node_data()
         data["node_id"] = self._id
+        trial = 0
+        while trial != 6:
+            try:
                 response = yield from self._compute.post("/projects/{}/{}/nodes".format(self._project.id, self._node_type), data=data)
+            except ComputeConflict as e:
+                if  e.response.get("exception") == "ImageMissingError":
+                    res = yield from self._upload_missing_image(self._node_type, e.response["image"])
+                    if not res:
+                        raise e
+            else:
                 self.parse_node_response(response.json)
+                return True
+            trial += 1
 
     @asyncio.coroutine
     def update(self, name=None, console=None, console_type=None, properties={}):
@@ -236,6 +251,22 @@ class Node:
         else:
             return (yield from self._compute.delete("/projects/{}/{}/nodes/{}{}".format(self._project.id, self._node_type, self._id, path)))
 
+    @asyncio.coroutine
+    def _upload_missing_image(self, type, img):
+        """
+        Search an image on local computer and upload it to remote compute
+        if the image exists
+        """
+        for directory in images_directories(type):
+            image = os.path.join(directory, img)
+            if os.path.exists(image):
+                self.project.controller.notification.emit("log.info", {"message": "Uploading missing image {}".format(img)})
+                with open(image, 'rb') as f:
+                    yield from self._compute.post("/{}/images/{}".format(self._node_type, os.path.basename(img)), data=f, timeout=None)
+                self.project.controller.notification.emit("log.info", {"message": "Upload finished for {}".format(img)})
+                return True
+        return False
+
     @asyncio.coroutine
     def dynamips_auto_idlepc(self):
         """

gns3server/utils/images.py

@@ -18,10 +18,44 @@
 import os
 import hashlib
 
+
+from ..config import Config
+from . import force_unix_path
+
+
 import logging
 log = logging.getLogger(__name__)
 
 
+def images_directories(type):
+    """
+    Return all directory where we will look for images
+    by priority
+
+    :param type: Type of emulator
+    """
+    server_config = Config.instance().get_section_config("Server")
+
+    paths = []
+    img_dir = os.path.expanduser(server_config.get("images_path", "~/GNS3/images"))
+    if type == "qemu":
+        type_img_directory = os.path.join(img_dir, "QEMU")
+    elif type == "iou":
+        type_img_directory = os.path.join(img_dir, "IOU")
+    elif type == "dynamips":
+        type_img_directory = os.path.join(img_dir, "IOS")
+    else:
+        raise NotImplementedError("%s is not supported", type)
+    os.makedirs(type_img_directory, exist_ok=True)
+    paths.append(type_img_directory)
+    for directory in server_config.get("additional_images_path", "").split(":"):
+        paths.append(directory)
+    # Compatibility with old topologies we look in parent directory
+    paths.append(img_dir)
+    # Return only the existings paths
+    return [force_unix_path(p) for p in paths if os.path.exists(p)]
+
+
 def md5sum(path):
     """
     Return the md5sum of an image and cache it on disk

gns3server/web/route.py

@@ -197,11 +197,16 @@ class Route(object):
                     response = Response(request=request, route=route)
                     response.set_status(409)
                     response.json({"message": str(e), "status": 409})
-                except (NodeError, UbridgeError, ImageMissingError) as e:
+                except (NodeError, UbridgeError) as e:
                     log.error("Node error detected: {type}".format(type=e.__class__.__name__), exc_info=1)
                     response = Response(request=request, route=route)
                     response.set_status(409)
                     response.json({"message": str(e), "status": 409, "exception": e.__class__.__name__})
+                except (ImageMissingError) as e:
+                    log.error("Image missing error detected: {}".format(e.image))
+                    response = Response(request=request, route=route)
+                    response.set_status(409)
+                    response.json({"message": str(e), "status": 409, "image": e.image, "exception": e.__class__.__name__})
                 except asyncio.futures.CancelledError as e:
                     log.error("Request canceled")
                     response = Response(request=request, route=route)

tests/compute/test_manager.py

@@ -88,28 +88,6 @@ def test_create_node_old_topology(loop, project, tmpdir, vpcs):
             assert f.read() == "1"
 
 
-def test_images_directories(qemu, tmpdir):
-    path1 = tmpdir / "images1" / "QEMU" / "test1.bin"
-    path1.write("1", ensure=True)
-    path1 = force_unix_path(str(path1))
-
-    path2 = tmpdir / "images2" / "test2.bin"
-    path2.write("1", ensure=True)
-    path2 = force_unix_path(str(path2))
-
-    with patch("gns3server.config.Config.get_section_config", return_value={
-            "images_path": str(tmpdir / "images1"),
-            "additional_images_path": "/tmp/null24564:{}".format(tmpdir / "images2"),
-            "local": False}):
-
-        # /tmp/null24564 is ignored because doesn't exists
-        res = qemu.images_directories()
-        assert res[0] == str(tmpdir / "images1" / "QEMU")
-        assert res[1] == str(tmpdir / "images2")
-        assert res[2] == str(tmpdir / "images1")
-        assert len(res) == 3
-
-
 def test_get_abs_image_path(qemu, tmpdir):
     os.makedirs(str(tmpdir / "QEMU"))
     path1 = force_unix_path(str(tmpdir / "test1.bin"))

tests/controller/test_compute.py

@@ -23,7 +23,7 @@ import asyncio
 from unittest.mock import patch, MagicMock
 
 from gns3server.controller.project import Project
-from gns3server.controller.compute import Compute, ComputeError
+from gns3server.controller.compute import Compute, ComputeError, ComputeConflict
 from gns3server.version import __version__
 from tests.utils import asyncio_patch, AsyncioMagicMock
 
@@ -139,9 +139,19 @@ def test_compute_httpQueryNotConnectedNonGNS3Server2(compute, async_run):
 def test_compute_httpQueryError(compute, async_run):
     response = MagicMock()
     with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
-        response.status = 409
+        response.status = 404
 
-        with pytest.raises(aiohttp.web.HTTPConflict):
+        with pytest.raises(aiohttp.web.HTTPNotFound):
+            async_run(compute.post("/projects", {"a": "b"}))
+
+
+def test_compute_httpQueryConflictError(compute, async_run):
+    response = MagicMock()
+    with asyncio_patch("aiohttp.ClientSession.request", return_value=response) as mock:
+        response.status = 409
+        response.read = AsyncioMagicMock(return_value=b'{"message": "Test"}')
+
+        with pytest.raises(ComputeConflict):
             async_run(compute.post("/projects", {"a": "b"}))
 
 

tests/controller/test_node.py

@@ -18,7 +18,8 @@
 import pytest
 import uuid
 import asyncio
-from unittest.mock import MagicMock
+import os
+from unittest.mock import MagicMock, ANY
 
 
 from tests.utils import AsyncioMagicMock
@@ -77,7 +78,7 @@ def test_create(node, compute, project, async_run):
     response.json = {"console": 2048}
     compute.post = AsyncioMagicMock(return_value=response)
 
-    async_run(node.create())
+    assert async_run(node.create()) is True
     data = {
         "console": 2048,
         "console_type": "vnc",
@@ -90,6 +91,28 @@ def test_create(node, compute, project, async_run):
     assert node._properties == {"startup_script": "echo test"}
 
 
+def test_create_image_missing(node, compute, project, async_run):
+    node._console = 2048
+
+    node.__calls = 0
+    @asyncio.coroutine
+    def resp(*args, **kwargs):
+        node.__calls += 1
+        response = MagicMock()
+        if node.__calls == 1:
+            response.status = 409
+            response.json = {"image": "linux.img", "exception": "ImageMissingError"}
+        else:
+            response.status = 200
+        return response
+
+    compute.post = AsyncioMagicMock(side_effect=resp)
+    node._upload_missing_image = AsyncioMagicMock(return_value=True)
+
+    assert async_run(node.create()) is True
+    node._upload_missing_image.called is True
+
+
 def test_update(node, compute, project, async_run):
     response = MagicMock()
     response.json = {"console": 2048}
@@ -193,3 +216,16 @@ def test_dynamips_idlepc_proposals(node, async_run, compute):
 
     async_run(node.dynamips_idlepc_proposals())
     compute.get.assert_called_with("/projects/{}/dynamips/nodes/{}/idlepc_proposals".format(node.project.id, node.id), timeout=240)
+
+
+def test_upload_missing_image(compute, controller, async_run, images_dir):
+    project = Project(str(uuid.uuid4()), controller=controller)
+    node = Node(project, compute,
+                name="demo",
+                node_id=str(uuid.uuid4()),
+                node_type="qemu",
+                properties={"hda_disk_image": "linux.img"})
+    open(os.path.join(images_dir, "linux.img"), 'w+').close()
+    assert async_run(node._upload_missing_image("qemu", "linux.img")) is True
+    compute.post.assert_called_with("/qemu/images/linux.img", data=ANY, timeout=None)
+

tests/utils.py

@@ -70,7 +70,10 @@ class AsyncioMagicMock(unittest.mock.MagicMock):
     Magic mock returning coroutine
     """
 
-    def __init__(self, return_value=None, **kwargs):
+    def __init__(self, return_value=None, return_values=None, **kwargs):
+        """
+        :return_values: Array of return value at each call will return the next
+        """
         if return_value:
             future = asyncio.Future()
             future.set_result(return_value)

tests/utils/test_images.py

@@ -16,8 +16,33 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+from unittest.mock import patch
 
-from gns3server.utils.images import md5sum, remove_checksum
+
+from gns3server.utils import force_unix_path
+from gns3server.utils.images import md5sum, remove_checksum, images_directories
+
+
+def test_images_directories(tmpdir):
+    path1 = tmpdir / "images1" / "QEMU" / "test1.bin"
+    path1.write("1", ensure=True)
+    path1 = force_unix_path(str(path1))
+
+    path2 = tmpdir / "images2" / "test2.bin"
+    path2.write("1", ensure=True)
+    path2 = force_unix_path(str(path2))
+
+    with patch("gns3server.config.Config.get_section_config", return_value={
+            "images_path": str(tmpdir / "images1"),
+            "additional_images_path": "/tmp/null24564:{}".format(tmpdir / "images2"),
+            "local": False}):
+
+        # /tmp/null24564 is ignored because doesn't exists
+        res = images_directories("qemu")
+        assert res[0] == str(tmpdir / "images1" / "QEMU")
+        assert res[1] == str(tmpdir / "images2")
+        assert res[2] == str(tmpdir / "images1")
+        assert len(res) == 3
 
 
 def test_md5sum(tmpdir):

tests/utils/test_path.py

@@ -19,7 +19,9 @@ import os
 import pytest
 import aiohttp
 
+
 from gns3server.utils.path import check_path_allowed, get_default_project_directory
+from gns3server.utils import force_unix_path
 
 
 def test_check_path_allowed(config, tmpdir):