From af780313229187eb2633684d57e3f45696002a77 Mon Sep 17 00:00:00 2001 From: Karim Date: Wed, 5 Jun 2019 09:39:44 +0100 Subject: [PATCH] Resolve conflicts in docker volumes instead of error. Fixes https://github.com/GNS3/gns3-server/issues/1595 --- gns3server/compute/docker/docker_vm.py | 21 ++- tests/compute/docker/test_docker_vm.py | 220 +++++++++++++++++++++++-- 2 files changed, 215 insertions(+), 26 deletions(-) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 27053ea4..d7d4b0dc 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -248,27 +248,32 @@ class DockerVM(BaseNode): # We mount our own etc/network try: - network_config = self._create_network_config() + self._create_network_config() except OSError as e: raise DockerError("Could not create network config in the container: {}".format(e)) - binds.append("{}:/gns3volumes/etc/network:rw".format(network_config)) + volumes = ["/etc/network"] - self._volumes = ["/etc/network"] - volumes = list((image_info.get("Config", {}).get("Volumes") or {}).keys()) + volumes.extend((image_info.get("Config", {}).get("Volumes") or {}).keys()) for volume in self._extra_volumes: if not volume.strip() or volume[0] != "/" or volume.find("..") >= 0: raise DockerError("Persistent volume '{}' has invalid format. It must start with a '/' and not contain '..'.".format(volume)) volumes.extend(self._extra_volumes) + + self._volumes = [] # define lambdas for validation checks nf = lambda x: re.sub(r"//+", "/", (x if x.endswith("/") else x + "/")) - incompatible = lambda v1, v2: nf(v1).startswith(nf(v2)) or nf(v2).startswith(nf(v1)) + generalises = lambda v1, v2: nf(v2).startswith(nf(v1)) for volume in volumes: - if [ v for v in self._volumes if incompatible(v, volume) ] : - raise DockerError("Duplicate persistent volume {} detected.\n\nVolumes specified in docker image as well as user specified persistent volumes must be unique.".format(volume)) + # remove any mount that is equal or more specific, then append this one + self._volumes = list(filter(lambda v: not generalises(volume, v), self._volumes)) + # if there is nothing more general, append this mount + if not [ v for v in self._volumes if generalises(v, volume) ] : + self._volumes.append(volume) + + for volume in self._volumes: source = os.path.join(self.working_dir, os.path.relpath(volume, "/")) os.makedirs(source, exist_ok=True) binds.append("{}:/gns3volumes{}".format(source, volume)) - self._volumes.append(volume) return binds diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 0f05fe59..da54bfbc 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -99,7 +99,7 @@ def test_create(loop, project, manager): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")) + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True }, @@ -138,7 +138,7 @@ def test_create_with_tag(loop, project, manager): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")) + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True }, @@ -180,7 +180,7 @@ def test_create_vnc(loop, project, manager): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")), + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), '/tmp/.X11-unix/:/tmp/.X11-unix/' ], "Privileged": True @@ -296,7 +296,7 @@ def test_create_start_cmd(loop, project, manager): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")) + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True }, @@ -396,7 +396,7 @@ def test_create_image_not_available(loop, project, manager): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")) + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True }, @@ -439,7 +439,7 @@ def test_create_with_user(loop, project, manager): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")) + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True }, @@ -509,8 +509,35 @@ def test_create_with_extra_volumes_duplicate_1_image(loop, project, manager): with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1"]) - with pytest.raises(DockerError): - loop.run_until_complete(asyncio.ensure_future(vm.create())) + loop.run_until_complete(asyncio.ensure_future(vm.create())) + mock.assert_called_with("POST", "containers/create", data={ + "Tty": True, + "OpenStdin": True, + "StdinOnce": False, + "HostConfig": + { + "CapAdd": ["ALL"], + "Binds": [ + "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), + "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")), + ], + "Privileged": True + }, + "Volumes": {}, + "NetworkDisabled": True, + "Name": "test", + "Hostname": "test", + "Image": "ubuntu:latest", + "Env": [ + "container=docker", + "GNS3_MAX_ETHERNET=eth0", + "GNS3_VOLUMES=/etc/network:/vol/1" + ], + "Entrypoint": ["/gns3/init.sh"], + "Cmd": ["/bin/sh"] + }) + assert vm._cid == "e90e34656806" def test_create_with_extra_volumes_duplicate_2_user(loop, project, manager): @@ -521,8 +548,35 @@ def test_create_with_extra_volumes_duplicate_2_user(loop, project, manager): with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1", "/vol/1"]) - with pytest.raises(DockerError): - loop.run_until_complete(asyncio.ensure_future(vm.create())) + loop.run_until_complete(asyncio.ensure_future(vm.create())) + mock.assert_called_with("POST", "containers/create", data={ + "Tty": True, + "OpenStdin": True, + "StdinOnce": False, + "HostConfig": + { + "CapAdd": ["ALL"], + "Binds": [ + "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), + "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")), + ], + "Privileged": True + }, + "Volumes": {}, + "NetworkDisabled": True, + "Name": "test", + "Hostname": "test", + "Image": "ubuntu:latest", + "Env": [ + "container=docker", + "GNS3_MAX_ETHERNET=eth0", + "GNS3_VOLUMES=/etc/network:/vol/1" + ], + "Entrypoint": ["/gns3/init.sh"], + "Cmd": ["/bin/sh"] + }) + assert vm._cid == "e90e34656806" def test_create_with_extra_volumes_duplicate_3_subdir(loop, project, manager): @@ -533,8 +587,35 @@ def test_create_with_extra_volumes_duplicate_3_subdir(loop, project, manager): with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol/1/", "/vol"]) - with pytest.raises(DockerError): - loop.run_until_complete(asyncio.ensure_future(vm.create())) + loop.run_until_complete(asyncio.ensure_future(vm.create())) + mock.assert_called_with("POST", "containers/create", data={ + "Tty": True, + "OpenStdin": True, + "StdinOnce": False, + "HostConfig": + { + "CapAdd": ["ALL"], + "Binds": [ + "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), + "{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")), + ], + "Privileged": True + }, + "Volumes": {}, + "NetworkDisabled": True, + "Name": "test", + "Hostname": "test", + "Image": "ubuntu:latest", + "Env": [ + "container=docker", + "GNS3_MAX_ETHERNET=eth0", + "GNS3_VOLUMES=/etc/network:/vol" + ], + "Entrypoint": ["/gns3/init.sh"], + "Cmd": ["/bin/sh"] + }) + assert vm._cid == "e90e34656806" def test_create_with_extra_volumes_duplicate_4_backslash(loop, project, manager): @@ -545,8 +626,111 @@ def test_create_with_extra_volumes_duplicate_4_backslash(loop, project, manager) with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/vol//", "/vol"]) - with pytest.raises(DockerError): - loop.run_until_complete(asyncio.ensure_future(vm.create())) + loop.run_until_complete(asyncio.ensure_future(vm.create())) + mock.assert_called_with("POST", "containers/create", data={ + "Tty": True, + "OpenStdin": True, + "StdinOnce": False, + "HostConfig": + { + "CapAdd": ["ALL"], + "Binds": [ + "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), + "{}:/gns3volumes/vol".format(os.path.join(vm.working_dir, "vol")), + ], + "Privileged": True + }, + "Volumes": {}, + "NetworkDisabled": True, + "Name": "test", + "Hostname": "test", + "Image": "ubuntu:latest", + "Env": [ + "container=docker", + "GNS3_MAX_ETHERNET=eth0", + "GNS3_VOLUMES=/etc/network:/vol" + ], + "Entrypoint": ["/gns3/init.sh"], + "Cmd": ["/bin/sh"] + }) + assert vm._cid == "e90e34656806" + +def test_create_with_extra_volumes_duplicate_5_subdir_issue_1595(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [], + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/etc"]) + loop.run_until_complete(asyncio.ensure_future(vm.create())) + mock.assert_called_with("POST", "containers/create", data={ + "Tty": True, + "OpenStdin": True, + "StdinOnce": False, + "HostConfig": + { + "CapAdd": ["ALL"], + "Binds": [ + "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")), + ], + "Privileged": True + }, + "Volumes": {}, + "NetworkDisabled": True, + "Name": "test", + "Hostname": "test", + "Image": "ubuntu:latest", + "Env": [ + "container=docker", + "GNS3_MAX_ETHERNET=eth0", + "GNS3_VOLUMES=/etc" + ], + "Entrypoint": ["/gns3/init.sh"], + "Cmd": ["/bin/sh"] + }) + assert vm._cid == "e90e34656806" + +def test_create_with_extra_volumes_duplicate_6_subdir_issue_1595(loop, project, manager): + + response = { + "Id": "e90e34656806", + "Warnings": [], + } + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu:latest", extra_volumes=["/etc/test", "/etc"]) + loop.run_until_complete(asyncio.ensure_future(vm.create())) + mock.assert_called_with("POST", "containers/create", data={ + "Tty": True, + "OpenStdin": True, + "StdinOnce": False, + "HostConfig": + { + "CapAdd": ["ALL"], + "Binds": [ + "{}:/gns3:ro".format(get_resource("compute/docker/resources")), + "{}:/gns3volumes/etc".format(os.path.join(vm.working_dir, "etc")), + ], + "Privileged": True + }, + "Volumes": {}, + "NetworkDisabled": True, + "Name": "test", + "Hostname": "test", + "Image": "ubuntu:latest", + "Env": [ + "container=docker", + "GNS3_MAX_ETHERNET=eth0", + "GNS3_VOLUMES=/etc" + ], + "Entrypoint": ["/gns3/init.sh"], + "Cmd": ["/bin/sh"] + }) + assert vm._cid == "e90e34656806" def test_create_with_extra_volumes(loop, project, manager): @@ -572,7 +756,7 @@ def test_create_with_extra_volumes(loop, project, manager): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")), + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "{}:/gns3volumes/vol/1".format(os.path.join(vm.working_dir, "vol", "1")), "{}:/gns3volumes/vol/2".format(os.path.join(vm.working_dir, "vol", "2")), ], @@ -795,7 +979,7 @@ def test_update(loop, vm): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")) + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True }, @@ -864,7 +1048,7 @@ def test_update_running(loop, vm): "CapAdd": ["ALL"], "Binds": [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")) + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")) ], "Privileged": True }, @@ -1138,7 +1322,7 @@ def test_mount_binds(vm, tmpdir): dst = os.path.join(vm.working_dir, "test/experimental") assert vm._mount_binds(image_infos) == [ "{}:/gns3:ro".format(get_resource("compute/docker/resources")), - "{}:/gns3volumes/etc/network:rw".format(os.path.join(vm.working_dir, "etc", "network")), + "{}:/gns3volumes/etc/network".format(os.path.join(vm.working_dir, "etc", "network")), "{}:/gns3volumes{}".format(dst, "/test/experimental") ]