From 0c19bc8d4307d84e20c944bee8d09b3fd9b55335 Mon Sep 17 00:00:00 2001 From: Karim Date: Mon, 22 Apr 2019 09:53:38 +0100 Subject: [PATCH 1/2] Support for additional persistent docker volumes to be specified within appliance configuration --- gns3server/compute/docker/docker_vm.py | 28 ++++-- .../handlers/api/compute/docker_handler.py | 5 +- gns3server/schemas/docker.py | 16 +++ tests/compute/docker/test_docker_vm.py | 99 +++++++++++++++++++ 4 files changed, 139 insertions(+), 9 deletions(-) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index e59321b0..9bf63c41 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -64,11 +64,12 @@ class DockerVM(BaseNode): :param console_http_port: Port to redirect HTTP queries :param console_http_path: Url part with the path of the web interface :param extra_hosts: Hosts which will be written into /etc/hosts into docker conainer + :param extra_volumes: Additional directories to make persistent """ def __init__(self, name, node_id, project, manager, image, console=None, aux=None, start_command=None, adapters=None, environment=None, console_type="telnet", console_resolution="1024x768", - console_http_port=80, console_http_path="/", extra_hosts=None): + console_http_port=80, console_http_path="/", extra_hosts=None, extra_volumes=[]): super().__init__(name, node_id, project, manager, console=console, aux=aux, allocate_aux=True, console_type=console_type) @@ -89,6 +90,7 @@ class DockerVM(BaseNode): self._console_http_port = console_http_port self._console_websocket = None self._extra_hosts = extra_hosts + self._extra_volumes = extra_volumes or [] self._permissions_fixed = False self._display = None self._closing = False @@ -125,7 +127,8 @@ class DockerVM(BaseNode): "status": self.status, "environment": self.environment, "node_directory": self.working_path, - "extra_hosts": self.extra_hosts + "extra_hosts": self.extra_hosts, + "extra_volumes": self.extra_volumes, } def _get_free_display_port(self): @@ -197,6 +200,14 @@ class DockerVM(BaseNode): def extra_hosts(self, extra_hosts): self._extra_hosts = extra_hosts + @property + def extra_volumes(self): + return self._extra_volumes + + @extra_volumes.setter + def extra_volumes(self, extra_volumes): + self._extra_volumes = extra_volumes + async def _get_container_state(self): """ Returns the container state (e.g. running, paused etc.) @@ -242,11 +253,14 @@ class DockerVM(BaseNode): binds.append("{}:/gns3volumes/etc/network:rw".format(network_config)) self._volumes = ["/etc/network"] - - volumes = image_info.get("Config", {}).get("Volumes") - if volumes is None: - return binds - for volume in volumes.keys(): + volumes = list((image_info.get("Config", {}).get("Volumes") or {}).keys()) + for volume in self._extra_volumes: + if not volume.strip() or volume[0] != "/": + raise DockerError("Additional volume '{}' has invalid format.".format(volume)) + volumes.extend(self._extra_volumes) + for volume in volumes: + if volume in self._volumes: + raise DockerError("Duplicate persistent volume {}".format(volume)) source = os.path.join(self.working_dir, os.path.relpath(volume, "/")) os.makedirs(source, exist_ok=True) binds.append("{}:/gns3volumes{}".format(source, volume)) diff --git a/gns3server/handlers/api/compute/docker_handler.py b/gns3server/handlers/api/compute/docker_handler.py index 9bd13848..83a5dd0d 100644 --- a/gns3server/handlers/api/compute/docker_handler.py +++ b/gns3server/handlers/api/compute/docker_handler.py @@ -61,7 +61,8 @@ class DockerHandler: console_http_port=request.json.get("console_http_port", 80), console_http_path=request.json.get("console_http_path", "/"), aux=request.json.get("aux"), - extra_hosts=request.json.get("extra_hosts")) + extra_hosts=request.json.get("extra_hosts"), + extra_volumes=request.json.get("extra_volumes")) for name, value in request.json.items(): if name != "node_id": if hasattr(container, name) and getattr(container, name) != value: @@ -316,7 +317,7 @@ class DockerHandler: props = [ "name", "console", "aux", "console_type", "console_resolution", "console_http_port", "console_http_path", "start_command", - "environment", "adapters", "extra_hosts" + "environment", "adapters", "extra_hosts", "extra_volumes" ] changed = False diff --git a/gns3server/schemas/docker.py b/gns3server/schemas/docker.py index d6d32bfd..6cea166a 100644 --- a/gns3server/schemas/docker.py +++ b/gns3server/schemas/docker.py @@ -95,6 +95,14 @@ DOCKER_CREATE_SCHEMA = { "type": ["string", "null"], "minLength": 0, }, + "extra_volumes": { + "description": "Additional directories to make persistent", + "type": "array", + "minItems": 0, + "items": { + "type": "string" + } + }, "container_id": { "description": "Docker container ID Read only", "type": "string", @@ -198,6 +206,14 @@ DOCKER_OBJECT_SCHEMA = { "type": ["string", "null"], "minLength": 0, }, + "extra_volumes": { + "description": "Additional directories to make persistent", + "type": "array", + "minItems": 0, + "items": { + "type": "string", + } + }, "node_directory": { "description": "Path to the node working directory Read only", "type": "string" diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 63dcb537..dc92fee9 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -62,6 +62,7 @@ def test_json(vm, project): 'console_http_port': 80, 'console_http_path': '/', 'extra_hosts': None, + 'extra_volumes': [], 'aux': vm.aux, 'start_command': vm.start_command, 'environment': vm.environment, @@ -458,6 +459,104 @@ def test_create_with_user(loop, project, manager): }) assert vm._cid == "e90e34656806" +def test_create_with_extra_volumes_invalid_format_1(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [] + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["vol1"]) + with pytest.raises(DockerError): + loop.run_until_complete(asyncio.ensure_future(vm.create())) + +def test_create_with_extra_volumes_invalid_format_2(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [] + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol1", ""]) + with pytest.raises(DockerError): + loop.run_until_complete(asyncio.ensure_future(vm.create())) + +def test_create_with_extra_volumes_duplicate_1_image(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [], + "Config" : { + "Volumes" : { + "/vol/1": None + }, + }, + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1"]) + with pytest.raises(DockerError): + loop.run_until_complete(asyncio.ensure_future(vm.create())) + +def test_create_with_extra_volumes_duplicate_2_user(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [], + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1", "/vol/1"]) + with pytest.raises(DockerError): + loop.run_until_complete(asyncio.ensure_future(vm.create())) + +def test_create_with_extra_volumes(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [], + "Config" : { + "Volumes" : { + "/vol/1": None + }, + }, + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/2"]) + loop.run_until_complete(asyncio.ensure_future(vm.create())) + mock.assert_called_with("POST", "containers/create", data={ + "Tty": True, + "OpenStdin": True, + "StdinOnce": False, + "HostConfig": + { + "CapAdd": ["ALL"], + "Binds": [ + "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")), + "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")), + "{}:/gns3volumes/vol/2".format(os.path.join(vm.working_dir, "vol", "2")), + ], + "Privileged": True + }, + "Volumes": {}, + "NetworkDisabled": True, + "Name": "test", + "Hostname": "test", + "Image": "ubuntu:latest", + "Env": [ + "container=docker", + "GNS3_MAX_ETHERNET=eth0", + "GNS3_VOLUMES=/etc/network:/vol/1:/vol/2" + ], + "Entrypoint": ["/gns3/init.sh"], + "Cmd": ["/bin/sh"] + }) + assert vm._cid == "e90e34656806" + def test_get_container_state(loop, vm): response = { "State": { From 81ddb0cfe647d4394f94a4bacb239e5bea7b74c2 Mon Sep 17 00:00:00 2001 From: Karim Date: Mon, 22 Apr 2019 12:46:28 +0100 Subject: [PATCH 2/2] Impoved docker volumes user input validation --- gns3server/compute/docker/docker_vm.py | 12 ++++++--- tests/compute/docker/test_docker_vm.py | 36 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 9bf63c41..27053ea4 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -26,6 +26,7 @@ import shlex import aiohttp import subprocess import os +import re from gns3server.utils.asyncio.telnet_server import AsyncioTelnetServer from gns3server.utils.asyncio.raw_command_server import AsyncioRawCommandServer @@ -255,12 +256,15 @@ class DockerVM(BaseNode): self._volumes = ["/etc/network"] volumes = list((image_info.get("Config", {}).get("Volumes") or {}).keys()) for volume in self._extra_volumes: - if not volume.strip() or volume[0] != "/": - raise DockerError("Additional volume '{}' has invalid format.".format(volume)) + if not volume.strip() or volume[0] != "/" or volume.find("..") >= 0: + raise DockerError("Persistent volume '{}' has invalid format. It must start with a '/' and not contain '..'.".format(volume)) volumes.extend(self._extra_volumes) + # define lambdas for validation checks + nf = lambda x: re.sub(r"//+", "/", (x if x.endswith("/") else x + "/")) + incompatible = lambda v1, v2: nf(v1).startswith(nf(v2)) or nf(v2).startswith(nf(v1)) for volume in volumes: - if volume in self._volumes: - raise DockerError("Duplicate persistent volume {}".format(volume)) + if [ v for v in self._volumes if incompatible(v, volume) ] : + raise DockerError("Duplicate persistent volume {} detected.\n\nVolumes specified in docker image as well as user specified persistent volumes must be unique.".format(volume)) source = os.path.join(self.working_dir, os.path.relpath(volume, "/")) os.makedirs(source, exist_ok=True) binds.append("{}:/gns3volumes{}".format(source, volume)) diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index dc92fee9..0f05fe59 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -483,6 +483,18 @@ def test_create_with_extra_volumes_invalid_format_2(loop, project, manager): with pytest.raises(DockerError): loop.run_until_complete(asyncio.ensure_future(vm.create())) +def test_create_with_extra_volumes_invalid_format_3(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [] + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol1/.."]) + with pytest.raises(DockerError): + loop.run_until_complete(asyncio.ensure_future(vm.create())) + def test_create_with_extra_volumes_duplicate_1_image(loop, project, manager): response = { @@ -512,6 +524,30 @@ def test_create_with_extra_volumes_duplicate_2_user(loop, project, manager): with pytest.raises(DockerError): loop.run_until_complete(asyncio.ensure_future(vm.create())) +def test_create_with_extra_volumes_duplicate_3_subdir(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [], + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1/", "/vol"]) + with pytest.raises(DockerError): + loop.run_until_complete(asyncio.ensure_future(vm.create())) + +def test_create_with_extra_volumes_duplicate_4_backslash(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [], + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol//", "/vol"]) + with pytest.raises(DockerError): + loop.run_until_complete(asyncio.ensure_future(vm.create())) + def test_create_with_extra_volumes(loop, project, manager): response = {