Merge pull request #1584 from kazkansouh/2.2-docker-volumes

Custom persistent docker volumes

gns3server/compute/docker/docker_vm.py

@@ -26,6 +26,7 @@ import shlex
 import aiohttp
 import subprocess
 import os
+import re
 
 from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer
 from gns3server.utils.asyncio.raw_command_server import AsyncioRawCommandServer
@@ -64,11 +65,12 @@ class DockerVM(BaseNode):
     :param console_http_port: Port to redirect HTTP queries
     :param console_http_path: Url part with the path of the web interface
     :param extra_hosts: Hosts which will be written into /etc/hosts into docker conainer
+    :param extra_volumes: Additional directories to make persistent
     """
 
     def __init__(self, name, node_id, project, manager, image, console=None, aux=None, start_command=None,
                  adapters=None, environment=None, console_type="telnet", console_resolution="1024x768",
-                 console_http_port=80, console_http_path="/", extra_hosts=None):
+                 console_http_port=80, console_http_path="/", extra_hosts=None, extra_volumes=[]):
 
         super().__init__(name, node_id, project, manager, console=console, aux=aux, allocate_aux=True, console_type=console_type)
 
@@ -89,6 +91,7 @@ class DockerVM(BaseNode):
         self._console_http_port = console_http_port
         self._console_websocket = None
         self._extra_hosts = extra_hosts
+        self._extra_volumes = extra_volumes or []
         self._permissions_fixed = False
         self._display = None
         self._closing = False
@@ -125,7 +128,8 @@ class DockerVM(BaseNode):
             "status": self.status,
             "environment": self.environment,
             "node_directory": self.working_path,
-            "extra_hosts": self.extra_hosts
+            "extra_hosts": self.extra_hosts,
+            "extra_volumes": self.extra_volumes,
         }
 
     def _get_free_display_port(self):
@@ -197,6 +201,14 @@ class DockerVM(BaseNode):
     def extra_hosts(self, extra_hosts):
         self._extra_hosts = extra_hosts
 
+    @property
+    def extra_volumes(self):
+        return self._extra_volumes
+
+    @extra_volumes.setter
+    def extra_volumes(self, extra_volumes):
+        self._extra_volumes = extra_volumes
+
     async def _get_container_state(self):
         """
         Returns the container state (e.g. running, paused etc.)
@@ -242,11 +254,17 @@ class DockerVM(BaseNode):
         binds.append("{}:/gns3volumes/etc/network:rw".format(network_config))
 
         self._volumes = ["/etc/network"]
-
+        volumes = list((image_info.get("Config", {}).get("Volumes") or {}).keys())
-        volumes = image_info.get("Config", {}).get("Volumes")
+        for volume in self._extra_volumes:
-        if volumes is None:
+            if not volume.strip() or volume[0] != "/" or volume.find("..") >= 0:
-            return binds
+                raise DockerError("Persistent volume '{}' has invalid format. It must start with a '/' and not contain '..'.".format(volume))
-        for volume in volumes.keys():
+        volumes.extend(self._extra_volumes)
+        # define lambdas for validation checks
+        nf = lambda x: re.sub(r"//+", "/", (x if x.endswith("/") else x + "/"))
+        incompatible = lambda v1, v2: nf(v1).startswith(nf(v2)) or nf(v2).startswith(nf(v1))
+        for volume in volumes:
+            if [ v for v in self._volumes if incompatible(v, volume) ] :
+                raise DockerError("Duplicate persistent volume {} detected.\n\nVolumes specified in docker image as well as user specified persistent volumes must be unique.".format(volume))
             source = os.path.join(self.working_dir, os.path.relpath(volume, "/"))
             os.makedirs(source, exist_ok=True)
             binds.append("{}:/gns3volumes{}".format(source, volume))

gns3server/handlers/api/compute/docker_handler.py

@@ -61,7 +61,8 @@ class DockerHandler:
                                                           console_http_port=request.json.get("console_http_port", 80),
                                                           console_http_path=request.json.get("console_http_path", "/"),
                                                           aux=request.json.get("aux"),
-                                                          extra_hosts=request.json.get("extra_hosts"))
+                                                          extra_hosts=request.json.get("extra_hosts"),
+                                                          extra_volumes=request.json.get("extra_volumes"))
         for name, value in request.json.items():
             if name != "node_id":
                 if hasattr(container, name) and getattr(container, name) != value:
@@ -316,7 +317,7 @@ class DockerHandler:
         props = [
             "name", "console", "aux", "console_type", "console_resolution",
             "console_http_port", "console_http_path", "start_command",
-            "environment", "adapters", "extra_hosts"
+            "environment", "adapters", "extra_hosts", "extra_volumes"
         ]
 
         changed = False

gns3server/schemas/docker.py

@@ -95,6 +95,14 @@ DOCKER_CREATE_SCHEMA = {
             "type": ["string", "null"],
             "minLength": 0,
         },
+        "extra_volumes": {
+            "description": "Additional directories to make persistent",
+            "type": "array",
+            "minItems": 0,
+            "items": {
+                "type": "string"
+            }
+        },
         "container_id": {
             "description": "Docker container ID Read only",
             "type": "string",
@@ -198,6 +206,14 @@ DOCKER_OBJECT_SCHEMA = {
             "type": ["string", "null"],
             "minLength": 0,
         },
+        "extra_volumes": {
+            "description": "Additional directories to make persistent",
+            "type": "array",
+            "minItems": 0,
+            "items": {
+                "type": "string",
+            }
+        },
         "node_directory": {
             "description": "Path to the node working directory  Read only",
             "type": "string"

tests/compute/docker/test_docker_vm.py

@@ -62,6 +62,7 @@ def test_json(vm, project):
         'console_http_port': 80,
         'console_http_path': '/',
         'extra_hosts': None,
+        'extra_volumes': [],
         'aux': vm.aux,
         'start_command': vm.start_command,
         'environment': vm.environment,
@@ -458,6 +459,140 @@ def test_create_with_user(loop, project, manager):
             })
         assert vm._cid == "e90e34656806"
 
+def test_create_with_extra_volumes_invalid_format_1(loop, project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": []
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images:
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["vol1"])
+            with pytest.raises(DockerError):
+                loop.run_until_complete(asyncio.ensure_future(vm.create()))
+
+def test_create_with_extra_volumes_invalid_format_2(loop, project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": []
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images:
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol1", ""])
+            with pytest.raises(DockerError):
+                loop.run_until_complete(asyncio.ensure_future(vm.create()))
+
+def test_create_with_extra_volumes_invalid_format_3(loop, project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": []
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images:
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol1/.."])
+            with pytest.raises(DockerError):
+                loop.run_until_complete(asyncio.ensure_future(vm.create()))
+
+def test_create_with_extra_volumes_duplicate_1_image(loop, project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": [],
+        "Config" : {
+            "Volumes" : {
+                "/vol/1": None
+            },
+        },
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images:
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1"])
+            with pytest.raises(DockerError):
+                loop.run_until_complete(asyncio.ensure_future(vm.create()))
+
+def test_create_with_extra_volumes_duplicate_2_user(loop, project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": [],
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images:
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1", "/vol/1"])
+            with pytest.raises(DockerError):
+                loop.run_until_complete(asyncio.ensure_future(vm.create()))
+
+def test_create_with_extra_volumes_duplicate_3_subdir(loop, project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": [],
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images:
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1/", "/vol"])
+            with pytest.raises(DockerError):
+                loop.run_until_complete(asyncio.ensure_future(vm.create()))
+
+def test_create_with_extra_volumes_duplicate_4_backslash(loop, project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": [],
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images:
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol//", "/vol"])
+            with pytest.raises(DockerError):
+                loop.run_until_complete(asyncio.ensure_future(vm.create()))
+
+def test_create_with_extra_volumes(loop, project, manager):
+
+    response = {
+        "Id": "e90e34656806",
+        "Warnings": [],
+        "Config" : {
+            "Volumes" : {
+                "/vol/1": None
+            },
+        },
+    }
+    with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images:
+        with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock:
+            vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/2"])
+            loop.run_until_complete(asyncio.ensure_future(vm.create()))
+            mock.assert_called_with("POST", "containers/create", data={
+                "Tty": True,
+                "OpenStdin": True,
+                "StdinOnce": False,
+                "HostConfig":
+                    {
+                        "CapAdd": ["ALL"],
+                        "Binds": [
+                            "{}:/gns3:ro".format(get_resource("compute/docker/resources")),
+                            "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")),
+                            "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")),
+                            "{}:/gns3volumes/vol/2".format(os.path.join(vm.working_dir, "vol", "2")),
+                        ],
+                        "Privileged": True
+                    },
+                "Volumes": {},
+                "NetworkDisabled": True,
+                "Name": "test",
+                "Hostname": "test",
+                "Image": "ubuntu:latest",
+                "Env": [
+                    "container=docker",
+                    "GNS3_MAX_ETHERNET=eth0",
+                    "GNS3_VOLUMES=/etc/network:/vol/1:/vol/2"
+                    ],
+                "Entrypoint": ["/gns3/init.sh"],
+                "Cmd": ["/bin/sh"]
+            })
+        assert vm._cid == "e90e34656806"
+
 def test_get_container_state(loop, vm):
     response = {
         "State": {