From 43081152ef3a962cce017cedb9a861f204ffd358 Mon Sep 17 00:00:00 2001 From: ziajka Date: Wed, 25 Apr 2018 16:03:01 +0200 Subject: [PATCH 01/10] Add support of ExtraHosts for Docker, Ref. gns3-gui#2482 --- gns3server/compute/docker/docker_vm.py | 24 ++++++++++++++++--- .../handlers/api/compute/docker_handler.py | 5 ++-- gns3server/schemas/docker.py | 10 ++++++++ tests/compute/docker/test_docker_vm.py | 22 +++++++++++++++++ tests/handlers/api/compute/test_docker.py | 8 ++++--- 5 files changed, 61 insertions(+), 8 deletions(-) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index d9f9f12d..4ba038b3 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -61,11 +61,12 @@ class DockerVM(BaseNode): :param console_resolution: Resolution of the VNC display :param console_http_port: Port to redirect HTTP queries :param console_http_path: Url part with the path of the web interface + :param extra_hosts: Hosts which will be written into /etc/hosts into docker conainer """ def __init__(self, name, node_id, project, manager, image, console=None, aux=None, start_command=None, adapters=None, environment=None, console_type="telnet", console_resolution="1024x768", - console_http_port=80, console_http_path="/"): + console_http_port=80, console_http_path="/", extra_hosts=None): super().__init__(name, node_id, project, manager, console=console, aux=aux, allocate_aux=True, console_type=console_type) @@ -84,6 +85,8 @@ class DockerVM(BaseNode): self._console_http_path = console_http_path self._console_http_port = console_http_port self._console_websocket = None + self._extra_hosts = extra_hosts + self._volumes = [] # Keep a list of created bridge self._bridges = set() @@ -114,7 +117,8 @@ class DockerVM(BaseNode): "start_command": self.start_command, "status": self.status, "environment": self.environment, - "node_directory": self.working_path + "node_directory": self.working_path, + "extra_hosts": self.extra_hosts } def _get_free_display_port(self): @@ -178,6 +182,14 @@ class DockerVM(BaseNode): def environment(self, command): self._environment = command + @property + def extra_hosts(self): + return self._extra_hosts + + @extra_hosts.setter + def extra_hosts(self, extra_hosts): + self._extra_hosts = extra_hosts + @asyncio.coroutine def _get_container_state(self): """Returns the container state (e.g. running, paused etc.) @@ -288,7 +300,7 @@ class DockerVM(BaseNode): "HostConfig": { "CapAdd": ["ALL"], "Privileged": True, - "Binds": self._mount_binds(image_infos) + "Binds": self._mount_binds(image_infos), }, "Volumes": {}, "Env": ["container=docker"], # Systemd compliant: https://github.com/GNS3/gns3-server/issues/573 @@ -325,6 +337,12 @@ class DockerVM(BaseNode): params["Env"].append("DISPLAY=:{}".format(self._display)) params["HostConfig"]["Binds"].append("/tmp/.X11-unix/:/tmp/.X11-unix/") + if self._extra_hosts is not None and self._extra_hosts.strip() != "": + params["HostConfig"]["ExtraHosts"] = [h.strip() + for h in self._extra_hosts.split("\n") + if h.strip() != "" ] + + result = yield from self.manager.query("POST", "containers/create", data=params) self._cid = result['Id'] log.info("Docker container '{name}' [{id}] created".format( diff --git a/gns3server/handlers/api/compute/docker_handler.py b/gns3server/handlers/api/compute/docker_handler.py index 139cd241..e3a9ee80 100644 --- a/gns3server/handlers/api/compute/docker_handler.py +++ b/gns3server/handlers/api/compute/docker_handler.py @@ -60,7 +60,8 @@ class DockerHandler: console_resolution=request.json.get("console_resolution", "1024x768"), console_http_port=request.json.get("console_http_port", 80), console_http_path=request.json.get("console_http_path", "/"), - aux=request.json.get("aux")) + aux=request.json.get("aux"), + extra_hosts=request.json.get("extra_hosts")) for name, value in request.json.items(): if name != "node_id": if hasattr(container, name) and getattr(container, name) != value: @@ -312,7 +313,7 @@ class DockerHandler: props = [ "name", "console", "aux", "console_type", "console_resolution", "console_http_port", "console_http_path", "start_command", - "environment", "adapters" + "environment", "adapters", "extra_hosts" ] changed = False diff --git a/gns3server/schemas/docker.py b/gns3server/schemas/docker.py index d9934d74..bdd856fd 100644 --- a/gns3server/schemas/docker.py +++ b/gns3server/schemas/docker.py @@ -87,6 +87,11 @@ DOCKER_CREATE_SCHEMA = { "type": ["string", "null"], "minLength": 0, }, + "extra_hosts": { + "description": "Docker extra hosts (added to /etc/hosts)", + "type": ["string", "null"], + "minLength": 0, + }, "container_id": { "description": "Docker container ID Read only", "type": "string", @@ -184,6 +189,11 @@ DOCKER_OBJECT_SCHEMA = { "type": ["string", "null"], "minLength": 0, }, + "extra_hosts": { + "description": "Docker extra hosts (added to /etc/hosts)", + "type": ["string", "null"], + "minLength": 0, + }, "node_directory": { "description": "Path to the node working directory Read only", "type": "string" diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index d48fcc98..60487ff6 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -202,6 +202,28 @@ def test_create_vnc(loop, project, manager): assert vm._console_type == "vnc" +def test_create_with_extra_hosts(loop, project, manager): + extra_hosts = "test:199.199.199.1\ntest2:199.199.199.1" + + response = { + "Id": "e90e34656806", + "Warnings": [] + } + + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu", extra_hosts=extra_hosts) + vm._start_vnc = MagicMock() + vm._display = 42 + loop.run_until_complete(asyncio.async(vm.create())) + called_kwargs = mock.call_args[1] + assert called_kwargs["data"]["HostConfig"]["ExtraHosts"] == [ + "test:199.199.199.1", + "test2:199.199.199.1" + ] + assert vm._extra_hosts == extra_hosts + + def test_create_start_cmd(loop, project, manager): response = { diff --git a/tests/handlers/api/compute/test_docker.py b/tests/handlers/api/compute/test_docker.py index 023eca8a..a5018703 100644 --- a/tests/handlers/api/compute/test_docker.py +++ b/tests/handlers/api/compute/test_docker.py @@ -32,7 +32,7 @@ pytestmark = pytest.mark.skipif(sys.platform.startswith("win"), reason="Not supp @pytest.fixture def base_params(): """Return standard parameters""" - return {"name": "PC TEST 1", "image": "nginx", "start_command": "nginx-daemon", "adapters": 2, "environment": "YES=1\nNO=0", "console_type": "telnet", "console_resolution": "1280x1024"} + return {"name": "PC TEST 1", "image": "nginx", "start_command": "nginx-daemon", "adapters": 2, "environment": "YES=1\nNO=0", "console_type": "telnet", "console_resolution": "1280x1024", "extra_hosts": "test:127.0.0.1"} @pytest.yield_fixture(autouse=True) @@ -69,7 +69,7 @@ def test_docker_create(http_compute, project, base_params): assert response.json["adapters"] == 2 assert response.json["environment"] == "YES=1\nNO=0" assert response.json["console_resolution"] == "1280x1024" - + assert response.json["extra_hosts"] == "test:127.0.0.1" def test_docker_start(http_compute, vm): with asyncio_patch("gns3server.compute.docker.docker_vm.DockerVM.start", return_value=True) as mock: @@ -150,7 +150,8 @@ def test_docker_update(http_compute, vm, tmpdir, free_console_port): response = http_compute.put("/projects/{project_id}/docker/nodes/{node_id}".format(project_id=vm["project_id"], node_id=vm["node_id"]), {"name": "test", "console": free_console_port, "start_command": "yes", - "environment": "GNS3=1\nGNS4=0"}, + "environment": "GNS3=1\nGNS4=0", + "extra_hosts": "test:127.0.0.1"}, example=True) assert mock.called assert response.status == 200 @@ -158,6 +159,7 @@ def test_docker_update(http_compute, vm, tmpdir, free_console_port): assert response.json["console"] == free_console_port assert response.json["start_command"] == "yes" assert response.json["environment"] == "GNS3=1\nGNS4=0" + assert response.json["extra_hosts"] == "test:127.0.0.1" def test_docker_start_capture(http_compute, vm, tmpdir, project): From 80958e8a6dac00f0f257e3ee09c658cd45625196 Mon Sep 17 00:00:00 2001 From: ziajka Date: Fri, 27 Apr 2018 14:33:07 +0200 Subject: [PATCH 02/10] support based on init.sh, Ref: #2482 --- gns3server/compute/docker/docker_vm.py | 23 +++++++++--- gns3server/compute/docker/resources/init.sh | 8 ++++ tests/compute/docker/test_docker_vm.py | 41 +++++++++++++++++---- 3 files changed, 60 insertions(+), 12 deletions(-) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 4ba038b3..6b7c3c7f 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -337,11 +337,10 @@ class DockerVM(BaseNode): params["Env"].append("DISPLAY=:{}".format(self._display)) params["HostConfig"]["Binds"].append("/tmp/.X11-unix/:/tmp/.X11-unix/") - if self._extra_hosts is not None and self._extra_hosts.strip() != "": - params["HostConfig"]["ExtraHosts"] = [h.strip() - for h in self._extra_hosts.split("\n") - if h.strip() != "" ] - + if self._extra_hosts: + extra_hosts = self._format_extra_hosts(self._extra_hosts) + if extra_hosts: + params["Env"].append("GNS3_EXTRA_HOSTS={}".format(extra_hosts)) result = yield from self.manager.query("POST", "containers/create", data=params) self._cid = result['Id'] @@ -349,6 +348,20 @@ class DockerVM(BaseNode): name=self._name, id=self._id)) return True + def _format_extra_hosts(self, extra_hosts): + lines = [h.strip() for h in self._extra_hosts.split("\n") if h.strip() != ""] + hosts = [] + try: + for host in lines: + hostname, ip = host.split(":") + hostname = hostname.strip() + ip = ip.strip() + if hostname and ip: + hosts.append((hostname, ip)) + except ValueError: + raise DockerError("Can't apply `ExtraHosts`, wrong format: {}".format(extra_hosts)) + return "\n".join(["{}\t{}".format(h[1], h[0]) for h in hosts]) + @asyncio.coroutine def update(self): """ diff --git a/gns3server/compute/docker/resources/init.sh b/gns3server/compute/docker/resources/init.sh index ee98bebd..9c317559 100755 --- a/gns3server/compute/docker/resources/init.sh +++ b/gns3server/compute/docker/resources/init.sh @@ -60,6 +60,14 @@ ff02::1 ip6-allnodes ff02::2 ip6-allrouters __EOF__ +# imitate docker's `ExtraHosts` behaviour +sed -i '/GNS3_EXTRA_HOSTS_START/,/GNS3_EXTRA_HOSTS_END/d' /etc/hosts +[ -n "$GNS3_EXTRA_HOSTS" ] && cat >> /etc/hosts << __EOF__ +# GNS3_EXTRA_HOSTS_START +$GNS3_EXTRA_HOSTS +# GNS3_EXTRA_HOSTS_END +__EOF__ + # configure loopback interface ip link set dev lo up diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 60487ff6..8360b2ff 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -61,6 +61,7 @@ def test_json(vm, project): 'console_resolution': '1024x768', 'console_http_port': 80, 'console_http_path': '/', + 'extra_hosts': None, 'aux': vm.aux, 'start_command': vm.start_command, 'environment': vm.environment, @@ -210,20 +211,46 @@ def test_create_with_extra_hosts(loop, project, manager): "Warnings": [] } - with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]) as mock_list_images: + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]): with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu", extra_hosts=extra_hosts) - vm._start_vnc = MagicMock() - vm._display = 42 loop.run_until_complete(asyncio.async(vm.create())) called_kwargs = mock.call_args[1] - assert called_kwargs["data"]["HostConfig"]["ExtraHosts"] == [ - "test:199.199.199.1", - "test2:199.199.199.1" - ] + assert "GNS3_EXTRA_HOSTS=199.199.199.1\ttest\n199.199.199.1\ttest2" in called_kwargs["data"]["Env"] assert vm._extra_hosts == extra_hosts +def test_create_with_extra_hosts_wrong_format(loop, project, manager): + extra_hosts = "test" + + response = { + "Id": "e90e34656806", + "Warnings": [] + } + + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]): + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response): + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu", extra_hosts=extra_hosts) + with pytest.raises(DockerError): + loop.run_until_complete(asyncio.async(vm.create())) + + +def test_create_with_empty_extra_hosts(loop, project, manager): + extra_hosts = "test:\n" + + response = { + "Id": "e90e34656806", + "Warnings": [] + } + + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]): + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu", extra_hosts=extra_hosts) + loop.run_until_complete(asyncio.async(vm.create())) + called_kwargs = mock.call_args[1] + assert len([ e for e in called_kwargs["data"]["Env"] if "GNS3_EXTRA_HOSTS" in e]) == 0 + + def test_create_start_cmd(loop, project, manager): response = { From e267f8a8b88603f0931ddcec172c2a400a65c942 Mon Sep 17 00:00:00 2001 From: ziajka Date: Fri, 4 May 2018 14:34:44 +0200 Subject: [PATCH 03/10] Project global variables --- .gitignore | 1 + gns3server/compute/docker/docker_vm.py | 4 ++ gns3server/compute/project.py | 14 +++++- gns3server/controller/project.py | 39 +++++++++++++++- gns3server/schemas/project.py | 46 +++++++++++++++++-- tests/compute/docker/test_docker_vm.py | 22 +++++++++ tests/compute/test_project.py | 22 ++++++++- tests/controller/test_project.py | 2 + tests/handlers/api/controller/test_project.py | 25 ++++++++++ 9 files changed, 167 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index e4ff9349..bf694717 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ nosetests.xml .project .pydevproject .settings +.vscode # Pycharm .idea diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 6b7c3c7f..7318b755 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -325,6 +325,10 @@ class DockerVM(BaseNode): # Give the information to the container the list of volume path mounted params["Env"].append("GNS3_VOLUMES={}".format(":".join(self._volumes))) + if self.project.variables: + for var in self.project.variables: + params["Env"].append("{}={}".format(var["name"], var.get('value', ''))) + if self._environment: for e in self._environment.strip().split("\n"): e = e.strip() diff --git a/gns3server/compute/project.py b/gns3server/compute/project.py index 8be8fb1e..3dc6723f 100644 --- a/gns3server/compute/project.py +++ b/gns3server/compute/project.py @@ -46,7 +46,7 @@ class Project: :param path: path of the project. (None use the standard directory) """ - def __init__(self, name=None, project_id=None, path=None): + def __init__(self, name=None, project_id=None, path=None, variables=None): self._name = name if project_id: @@ -61,6 +61,7 @@ class Project: self._nodes = set() self._used_tcp_ports = set() self._used_udp_ports = set() + self._variables = variables if path is None: location = get_default_project_directory() @@ -83,7 +84,8 @@ class Project: return { "name": self._name, - "project_id": self._id + "project_id": self._id, + "variables": self._variables } def _config(self): @@ -131,6 +133,14 @@ class Project: return self._nodes + @property + def variables(self): + return self._variables + + @variables.setter + def variables(self, variables): + self._variables = variables + def record_tcp_port(self, port): """ Associate a reserved TCP port number with this project. diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 24306cb6..ceee6939 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -68,7 +68,7 @@ class Project: def __init__(self, name=None, project_id=None, path=None, controller=None, status="opened", filename=None, auto_start=False, auto_open=False, auto_close=True, scene_height=1000, scene_width=2000, zoom=100, show_layers=False, snap_to_grid=False, show_grid=False, - grid_size=0, show_interface_labels=False): + grid_size=0, show_interface_labels=False, variables=None, supplier=None): self._controller = controller assert name is not None @@ -85,6 +85,9 @@ class Project: self._show_grid = show_grid self._grid_size = grid_size self._show_interface_labels = show_interface_labels + self._variables = variables + self._supplier = supplier + self._loading = False # Disallow overwrite of existing project @@ -266,6 +269,36 @@ class Project: """ self._show_interface_labels = show_interface_labels + @property + def variables(self): + """ + Variables applied to the project + :return: list + """ + return self._variables + + @variables.setter + def variables(self, variables): + """ + Setter for variables applied to the project + """ + self._variables = variables + + @property + def supplier(self): + """ + Supplier of the project + :return: dict + """ + return self._supplier + + @supplier.setter + def supplier(self, supplier): + """ + Setter for supplier of the project + """ + self._supplier = supplier + @property def auto_start(self): """ @@ -1012,7 +1045,9 @@ class Project: "snap_to_grid": self._snap_to_grid, "show_grid": self._show_grid, "grid_size": self._grid_size, - "show_interface_labels": self._show_interface_labels + "show_interface_labels": self._show_interface_labels, + "supplier": self._supplier, + "variables": self._variables } def __repr__(self): diff --git a/gns3server/schemas/project.py b/gns3server/schemas/project.py index 242f5e5b..c232f8ce 100644 --- a/gns3server/schemas/project.py +++ b/gns3server/schemas/project.py @@ -15,6 +15,40 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +SUPPLIER_OBJECT_SCHEMA = { + "type": ["object", "null"], + "description": "Supplier of the project", + "properties": { + "logo": { + "type": "string", + "description": "Path to the project supplier logo" + }, + "url": { + "type": "string", + "description": "URL to the project supplier site" + } + } +} + + +VARIABLES_OBJECT_SCHEMA = { + "type": ["array", "null"], + "description": "Variables required to run the project", + "items": { + "properties": { + "name": { + "type": "string", + "description": "Variable name" + }, + "value": { + "type": "string", + "description": "Variable value" + } + }, + "required": ["name"] + } +} + PROJECT_CREATE_SCHEMA = { "$schema": "http://json-schema.org/draft-04/schema#", @@ -73,7 +107,9 @@ PROJECT_CREATE_SCHEMA = { "show_interface_labels": { "type": "boolean", "description": "Show interface labels on the drawing area" - } + }, + "supplier": SUPPLIER_OBJECT_SCHEMA, + "variables": VARIABLES_OBJECT_SCHEMA }, "additionalProperties": False, "required": ["name"] @@ -136,7 +172,9 @@ PROJECT_UPDATE_SCHEMA = { "show_interface_labels": { "type": "boolean", "description": "Show interface labels on the drawing area" - } + }, + "supplier": SUPPLIER_OBJECT_SCHEMA, + "variables": VARIABLES_OBJECT_SCHEMA }, "additionalProperties": False, } @@ -215,7 +253,9 @@ PROJECT_OBJECT_SCHEMA = { "show_interface_labels": { "type": "boolean", "description": "Show interface labels on the drawing area" - } + }, + "supplier": SUPPLIER_OBJECT_SCHEMA, + "variables": VARIABLES_OBJECT_SCHEMA }, "additionalProperties": False, "required": ["project_id"] diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 8360b2ff..8e2075c5 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -251,6 +251,28 @@ def test_create_with_empty_extra_hosts(loop, project, manager): assert len([ e for e in called_kwargs["data"]["Env"] if "GNS3_EXTRA_HOSTS" in e]) == 0 +def test_create_with_project_variables(loop, project_with_variables, manager): + response = { + "Id": "e90e34656806", + "Warnings": [] + } + + project.variables = [ + {"name": "VAR1"}, + {"name": "VAR2", "value": "VAL1"} + ] + + with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]): + with asyncio_patch("gns3server.compute.docker.Docker.query", return_value=response) as mock: + vm = DockerVM("test", str(uuid.uuid4()), project, manager, "ubuntu") + loop.run_until_complete(asyncio.async(vm.create())) + called_kwargs = mock.call_args[1] + assert "VAR1=" in called_kwargs["data"]["Env"] + assert "VAR2=VAL1" in called_kwargs["data"]["Env"] + assert vm._extra_hosts == extra_hosts + + project.variables = None + def test_create_start_cmd(loop, project, manager): response = { diff --git a/tests/compute/test_project.py b/tests/compute/test_project.py index 1b349354..5713ebf4 100644 --- a/tests/compute/test_project.py +++ b/tests/compute/test_project.py @@ -92,9 +92,29 @@ def test_changing_path_not_allowed(tmpdir): p.path = str(tmpdir) +def test_variables(tmpdir): + variables = [{"name": "VAR1", "value": "VAL1"}] + p = Project(project_id=str(uuid4()), variables=variables) + assert p.variables == variables + + def test_json(tmpdir): p = Project(project_id=str(uuid4())) - assert p.__json__() == {"name": p.name, "project_id": p.id} + assert p.__json__() == { + "name": p.name, + "project_id": p.id, + "variables": None + } + + +def test_json_with_variables(tmpdir): + variables = [{"name": "VAR1", "value": "VAL1"}] + p = Project(project_id=str(uuid4()), variables=variables) + assert p.__json__() == { + "name": p.name, + "project_id": p.id, + "variables": variables + } def test_node_working_directory(tmpdir, node): diff --git a/tests/controller/test_project.py b/tests/controller/test_project.py index 77f8e799..6d0550ab 100644 --- a/tests/controller/test_project.py +++ b/tests/controller/test_project.py @@ -77,6 +77,8 @@ def test_json(tmpdir): "show_layers": False, "snap_to_grid": False, "grid_size": 0, + "supplier": None, + "variables": [] } diff --git a/tests/handlers/api/controller/test_project.py b/tests/handlers/api/controller/test_project.py index e75511c5..4e70f04d 100644 --- a/tests/handlers/api/controller/test_project.py +++ b/tests/handlers/api/controller/test_project.py @@ -67,6 +67,31 @@ def test_create_project_with_uuid(http_controller): assert response.json["name"] == "test" +def test_create_project_with_variables(http_controller): + variables = [ + {"name": "TEST1"}, + {"name": "TEST2", "value": "value1"} + ] + query = {"name": "test", "project_id": "30010203-0405-0607-0809-0a0b0c0d0e0f", "variables": variables} + response = http_controller.post("/projects", query) + assert response.status == 201 + assert response.json["variables"] == [ + {"name": "TEST1"}, + {"name": "TEST2", "value": "value1"} + ] + + +def test_create_project_with_supplier(http_controller): + supplier = { + 'logo': 'logo.png', + 'url': 'http://example.com' + } + query = {"name": "test", "project_id": "30010203-0405-0607-0809-0a0b0c0d0e0f", "supplier": supplier} + response = http_controller.post("/projects", query) + assert response.status == 201 + assert response.json["supplier"] == supplier + + def test_update_project(http_controller): query = {"name": "test", "project_id": "10010203-0405-0607-0809-0a0b0c0d0e0f"} response = http_controller.post("/projects", query) From 132c7c8f5b0b195ad4c1be1025da16abafa8571f Mon Sep 17 00:00:00 2001 From: ziajka Date: Mon, 7 May 2018 12:55:32 +0200 Subject: [PATCH 04/10] Fix tests --- tests/controller/test_project.py | 2 +- tests/handlers/api/compute/test_project.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/controller/test_project.py b/tests/controller/test_project.py index 6d0550ab..5c788a1b 100644 --- a/tests/controller/test_project.py +++ b/tests/controller/test_project.py @@ -78,7 +78,7 @@ def test_json(tmpdir): "snap_to_grid": False, "grid_size": 0, "supplier": None, - "variables": [] + "variables": None } diff --git a/tests/handlers/api/compute/test_project.py b/tests/handlers/api/compute/test_project.py index a1e1cd42..9db5350a 100644 --- a/tests/handlers/api/compute/test_project.py +++ b/tests/handlers/api/compute/test_project.py @@ -21,9 +21,6 @@ This test suite check /project endpoint import uuid import os -import asyncio -import aiohttp -import zipfile from unittest.mock import patch from tests.utils import asyncio_patch @@ -60,9 +57,10 @@ def test_show_project(http_compute): response = http_compute.post("/projects", query) assert response.status == 201 response = http_compute.get("/projects/40010203-0405-0607-0809-0a0b0c0d0e02", example=True) - assert len(response.json.keys()) == 2 + assert len(response.json.keys()) == 3 assert response.json["project_id"] == "40010203-0405-0607-0809-0a0b0c0d0e02" assert response.json["name"] == "test" + assert response.json["variables"] is None def test_show_project_invalid_uuid(http_compute): From e4436096c229d401aa43ea3e06dbecf93e2267c1 Mon Sep 17 00:00:00 2001 From: ziajka Date: Mon, 7 May 2018 15:18:19 +0200 Subject: [PATCH 05/10] Fix tests --- tests/compute/docker/test_docker_vm.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 8e2075c5..1fb6012b 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -251,7 +251,7 @@ def test_create_with_empty_extra_hosts(loop, project, manager): assert len([ e for e in called_kwargs["data"]["Env"] if "GNS3_EXTRA_HOSTS" in e]) == 0 -def test_create_with_project_variables(loop, project_with_variables, manager): +def test_create_with_project_variables(loop, project, manager): response = { "Id": "e90e34656806", "Warnings": [] @@ -269,10 +269,9 @@ def test_create_with_project_variables(loop, project_with_variables, manager): called_kwargs = mock.call_args[1] assert "VAR1=" in called_kwargs["data"]["Env"] assert "VAR2=VAL1" in called_kwargs["data"]["Env"] - assert vm._extra_hosts == extra_hosts - project.variables = None + def test_create_start_cmd(loop, project, manager): response = { From f2700ed44557dace9125d9fce2356f9d8ed9edc0 Mon Sep 17 00:00:00 2001 From: ziajka Date: Tue, 8 May 2018 10:07:42 +0200 Subject: [PATCH 06/10] Support of supplier and variables in topology --- gns3server/controller/topology.py | 2 + gns3server/schemas/topology.py | 4 ++ tests/controller/test_topology.py | 72 +++++++++++++++++++ tests/handlers/api/controller/test_project.py | 14 ++++ 4 files changed, 92 insertions(+) diff --git a/gns3server/controller/topology.py b/gns3server/controller/topology.py index efee8377..475c8c41 100644 --- a/gns3server/controller/topology.py +++ b/gns3server/controller/topology.py @@ -85,6 +85,8 @@ def project_to_topology(project): "show_grid": project.show_grid, "grid_size": project.grid_size, "show_interface_labels": project.show_interface_labels, + "variables": project.variables, + "supplier": project.supplier, "topology": { "nodes": [], "links": [], diff --git a/gns3server/schemas/topology.py b/gns3server/schemas/topology.py index faadf81b..7b319253 100644 --- a/gns3server/schemas/topology.py +++ b/gns3server/schemas/topology.py @@ -23,6 +23,8 @@ from gns3server.schemas.compute import COMPUTE_OBJECT_SCHEMA from gns3server.schemas.drawing import DRAWING_OBJECT_SCHEMA from gns3server.schemas.link import LINK_OBJECT_SCHEMA from gns3server.schemas.node import NODE_OBJECT_SCHEMA +from gns3server.schemas.project import VARIABLES_OBJECT_SCHEMA +from gns3server.schemas.project import SUPPLIER_OBJECT_SCHEMA TOPOLOGY_SCHEMA = { @@ -97,6 +99,8 @@ TOPOLOGY_SCHEMA = { "type": "boolean", "description": "Show interface labels on the drawing area" }, + "supplier": SUPPLIER_OBJECT_SCHEMA, + "variables": VARIABLES_OBJECT_SCHEMA, "topology": { "description": "The topology content", "type": "object", diff --git a/tests/controller/test_topology.py b/tests/controller/test_topology.py index b5d4b12f..0bf8a0cb 100644 --- a/tests/controller/test_topology.py +++ b/tests/controller/test_topology.py @@ -81,6 +81,26 @@ def test_basic_topology(tmpdir, async_run, controller): assert topo["topology"]["drawings"][0] == drawing.__json__(topology_dump=True) +def test_project_to_topology(tmpdir, controller): + variables = [ + {"name": "TEST1"}, + {"name": "TEST2", "value": "value1"} + ] + supplier = { + 'logo': 'logo.png', + 'url': 'http://example.com' + } + + project = Project(name="Test", controller=controller) + compute = Compute("my_compute", controller) + compute.http_query = MagicMock() + project.variables = variables + project.supplier = supplier + topo = project_to_topology(project) + assert topo["variables"] == variables + assert topo["supplier"] == supplier + + def test_load_topology(tmpdir): data = { "project_id": "69f26504-7aa3-48aa-9f29-798d44841211", @@ -137,3 +157,55 @@ def test_load_newer_topology(tmpdir): json.dump(data, f) with pytest.raises(aiohttp.web.HTTPConflict): topo = load_topology(path) + + +def test_load_topology_with_variables(tmpdir): + variables = [ + {"name": "TEST1"}, + {"name": "TEST2", "value": "value1"} + ] + data = { + "project_id": "69f26504-7aa3-48aa-9f29-798d44841211", + "name": "Test", + "revision": GNS3_FILE_FORMAT_REVISION, + "topology": { + "nodes": [], + "links": [], + "computes": [], + "drawings": [] + }, + "variables": variables, + "type": "topology", + "version": __version__} + + path = str(tmpdir / "test.gns3") + with open(path, "w+") as f: + json.dump(data, f) + topo = load_topology(path) + assert topo == data + + +def test_load_topology_with_supplier(tmpdir): + supplier = { + 'logo': 'logo.png', + 'url': 'http://example.com' + } + data = { + "project_id": "69f26504-7aa3-48aa-9f29-798d44841211", + "name": "Test", + "revision": GNS3_FILE_FORMAT_REVISION, + "topology": { + "nodes": [], + "links": [], + "computes": [], + "drawings": [] + }, + "supplier": supplier, + "type": "topology", + "version": __version__} + + path = str(tmpdir / "test.gns3") + with open(path, "w+") as f: + json.dump(data, f) + topo = load_topology(path) + assert topo == data \ No newline at end of file diff --git a/tests/handlers/api/controller/test_project.py b/tests/handlers/api/controller/test_project.py index 4e70f04d..b0c852fa 100644 --- a/tests/handlers/api/controller/test_project.py +++ b/tests/handlers/api/controller/test_project.py @@ -104,6 +104,20 @@ def test_update_project(http_controller): assert response.json["name"] == "test2" +def test_update_project_with_variables(http_controller): + variables = [ + {"name": "TEST1"}, + {"name": "TEST2", "value": "value1"} + ] + query = {"name": "test", "project_id": "10010203-0405-0607-0809-0a0b0c0d0e0f", "variables": variables} + response = http_controller.post("/projects", query) + assert response.status == 201 + query = {"name": "test2"} + response = http_controller.put("/projects/10010203-0405-0607-0809-0a0b0c0d0e0f", query, example=True) + assert response.status == 200 + assert response.json["variables"] == variables + + def test_list_projects(http_controller, tmpdir): http_controller.post("/projects", {"name": "test", "path": str(tmpdir), "project_id": "00010203-0405-0607-0809-0a0b0c0d0e0f"}) response = http_controller.get("/projects", example=True) From 0bcc657bf1ab10f38a1f3db6212b0ed025d81818 Mon Sep 17 00:00:00 2001 From: ziajka Date: Tue, 8 May 2018 16:22:35 +0200 Subject: [PATCH 07/10] Don't clean logo images when applied to the project --- gns3server/controller/project.py | 9 +++++++++ tests/controller/test_project.py | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index ceee6939..59bcfedc 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -725,6 +725,15 @@ class Project: except KeyError: pass + # don't remove supplier's logo + if self.supplier: + try: + logo = self.supplier['logo'] + pictures.remove(logo) + except KeyError: + pass + + for pict in pictures: os.remove(os.path.join(self.pictures_directory, pict)) except OSError as e: diff --git a/tests/controller/test_project.py b/tests/controller/test_project.py index 5c788a1b..1e76d39e 100644 --- a/tests/controller/test_project.py +++ b/tests/controller/test_project.py @@ -397,6 +397,26 @@ def test_clean_pictures(async_run, project, controller): assert not os.path.exists(os.path.join(project.pictures_directory, "test2.png")) +def test_clean_pictures_and_keep_supplier_logo(async_run, project, controller): + """ + When a project is close old pictures should be removed + """ + project.supplier = { + 'logo': 'logo.png' + } + + drawing = async_run(project.add_drawing()) + drawing._svg = "test.png" + open(os.path.join(project.pictures_directory, "test.png"), "w+").close() + open(os.path.join(project.pictures_directory, "test2.png"), "w+").close() + open(os.path.join(project.pictures_directory, "logo.png"), "w+").close() + + async_run(project.close()) + assert os.path.exists(os.path.join(project.pictures_directory, "test.png")) + assert not os.path.exists(os.path.join(project.pictures_directory, "test2.png")) + assert os.path.exists(os.path.join(project.pictures_directory, "logo.png")) + + def test_delete(async_run, project, controller): assert os.path.exists(project.path) async_run(project.delete()) From 83445214beb11af6341d84707ef245f49fdbf984 Mon Sep 17 00:00:00 2001 From: ziajka Date: Wed, 9 May 2018 11:25:55 +0200 Subject: [PATCH 08/10] Support for nested global variables --- gns3server/compute/docker/docker_vm.py | 18 ++++++++++++++---- tests/compute/docker/test_docker_vm.py | 4 +++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/gns3server/compute/docker/docker_vm.py b/gns3server/compute/docker/docker_vm.py index 7318b755..66e5d6f7 100644 --- a/gns3server/compute/docker/docker_vm.py +++ b/gns3server/compute/docker/docker_vm.py @@ -325,15 +325,20 @@ class DockerVM(BaseNode): # Give the information to the container the list of volume path mounted params["Env"].append("GNS3_VOLUMES={}".format(":".join(self._volumes))) - if self.project.variables: - for var in self.project.variables: - params["Env"].append("{}={}".format(var["name"], var.get('value', ''))) + variables = self.project.variables + if not variables: + variables = [] + + for var in variables: + formatted = self._format_env(variables, var.get('value', '')) + params["Env"].append("{}={}".format(var["name"], formatted)) if self._environment: for e in self._environment.strip().split("\n"): e = e.strip() if not e.startswith("GNS3_"): - params["Env"].append(e) + formatted = self._format_env(variables, e) + params["Env"].append(formatted) if self._console_type == "vnc": yield from self._start_vnc() @@ -352,6 +357,11 @@ class DockerVM(BaseNode): name=self._name, id=self._id)) return True + def _format_env(self, variables, env): + for variable in variables: + env = env.replace('${' + variable["name"] + '}', variable.get("value", "")) + return env + def _format_extra_hosts(self, extra_hosts): lines = [h.strip() for h in self._extra_hosts.split("\n") if h.strip() != ""] hosts = [] diff --git a/tests/compute/docker/test_docker_vm.py b/tests/compute/docker/test_docker_vm.py index 1fb6012b..f51e8c9c 100644 --- a/tests/compute/docker/test_docker_vm.py +++ b/tests/compute/docker/test_docker_vm.py @@ -259,7 +259,8 @@ def test_create_with_project_variables(loop, project, manager): project.variables = [ {"name": "VAR1"}, - {"name": "VAR2", "value": "VAL1"} + {"name": "VAR2", "value": "VAL1"}, + {"name": "VAR3", "value": "2x${VAR2}"} ] with asyncio_patch("gns3server.compute.docker.Docker.list_images", return_value=[{"image": "ubuntu"}]): @@ -269,6 +270,7 @@ def test_create_with_project_variables(loop, project, manager): called_kwargs = mock.call_args[1] assert "VAR1=" in called_kwargs["data"]["Env"] assert "VAR2=VAL1" in called_kwargs["data"]["Env"] + assert "VAR3=2xVAL1" in called_kwargs["data"]["Env"] project.variables = None From 7b3d5ae5e35b366da43233e9f64df9de9b023791 Mon Sep 17 00:00:00 2001 From: ziajka Date: Wed, 9 May 2018 15:29:35 +0200 Subject: [PATCH 09/10] Create/update project on compute when variables changes --- gns3server/compute/project.py | 13 +++++++++- gns3server/compute/project_manager.py | 7 +++--- gns3server/controller/compute.py | 3 ++- gns3server/controller/project.py | 13 +++++++++- .../handlers/api/compute/project_handler.py | 22 +++++++++++++++- tests/compute/test_project.py | 7 ++++++ tests/controller/test_project.py | 25 ++++++++++++++++--- tests/controller/test_topology.py | 2 ++ tests/handlers/api/compute/test_project.py | 17 +++++++++++++ 9 files changed, 98 insertions(+), 11 deletions(-) diff --git a/gns3server/compute/project.py b/gns3server/compute/project.py index 3dc6723f..3792a8a7 100644 --- a/gns3server/compute/project.py +++ b/gns3server/compute/project.py @@ -25,13 +25,13 @@ import zipfile import json from uuid import UUID, uuid4 + from .port_manager import PortManager from .notification_manager import NotificationManager from ..config import Config from ..utils.asyncio import wait_run_in_executor from ..utils.path import check_path_allowed, get_default_project_directory - import logging log = logging.getLogger(__name__) @@ -297,6 +297,17 @@ class Project: yield from node.delete() self._nodes.remove(node) + @asyncio.coroutine + def update(self, variables=None, **kwargs): + original_variables = self.variables + self.variables = variables + + # we need to update docker nodes when variables changes + if original_variables != variables: + for node in self.nodes: + if hasattr(node, 'update'): + yield from node.update() + @asyncio.coroutine def close(self): """ diff --git a/gns3server/compute/project_manager.py b/gns3server/compute/project_manager.py index 28c517d1..9abda38a 100644 --- a/gns3server/compute/project_manager.py +++ b/gns3server/compute/project_manager.py @@ -16,6 +16,7 @@ # along with this program. If not, see . import aiohttp +import asyncio import psutil import platform from .project import Project @@ -95,16 +96,16 @@ class ProjectManager: log.warning(message) project.emit("log.warning", {"message": message}) - def create_project(self, name=None, project_id=None, path=None): + def create_project(self, name=None, project_id=None, path=None, variables=None): """ Create a project and keep a references to it in project manager. See documentation of Project for arguments """ - if project_id is not None and project_id in self._projects: return self._projects[project_id] - project = Project(name=name, project_id=project_id, path=path) + project = Project(name=name, project_id=project_id, + path=path, variables=variables) self._check_available_disk_space(project) self._projects[project.id] = project return project diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index e1948a35..3bdee2f9 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -460,11 +460,12 @@ class Compute: msg = json.loads(response.data) action = msg.pop("action") event = msg.pop("event") - if action == "ping": self._cpu_usage_percent = event["cpu_usage_percent"] self._memory_usage_percent = event["memory_usage_percent"] self._controller.notification.emit("compute.updated", self.__json__()) + elif action == 'project.updated': + print(event) else: yield from self._controller.notification.dispatch(action, event, compute_id=self.id) if self._ws: diff --git a/gns3server/controller/project.py b/gns3server/controller/project.py index 59bcfedc..576f64df 100644 --- a/gns3server/controller/project.py +++ b/gns3server/controller/project.py @@ -136,6 +136,15 @@ class Project: self.controller.notification.emit("project.updated", self.__json__()) self.dump() + # update on computes + for compute in list(self._project_created_on_compute): + yield from compute.put( + "/projects/{}".format(self._id), { + "variables": self.variables + } + ) + + def reset(self): """ Called when open/close a project. Cleanup internal stuff @@ -493,12 +502,14 @@ class Project: yield from compute.post("/projects", data={ "name": self._name, "project_id": self._id, - "path": self._path + "path": self._path, + "variables": self._variables }) else: yield from compute.post("/projects", data={ "name": self._name, "project_id": self._id, + "variables": self._variables }) self._project_created_on_compute.add(compute) diff --git a/gns3server/handlers/api/compute/project_handler.py b/gns3server/handlers/api/compute/project_handler.py index a7d8f760..e850a955 100644 --- a/gns3server/handlers/api/compute/project_handler.py +++ b/gns3server/handlers/api/compute/project_handler.py @@ -73,11 +73,31 @@ class ProjectHandler: p = pm.create_project( name=request.json.get("name"), path=request.json.get("path"), - project_id=request.json.get("project_id") + project_id=request.json.get("project_id"), + variables=request.json.get("variables", None) ) response.set_status(201) response.json(p) + @Route.put( + r"/projects/{project_id}", + description="Update the project on the server", + status_codes={ + 201: "Project updated", + 403: "Forbidden to update a project" + }, + output=PROJECT_OBJECT_SCHEMA, + input=PROJECT_UPDATE_SCHEMA) + def update_project(request, response): + + pm = ProjectManager.instance() + project = pm.get_project(request.match_info["project_id"]) + yield from project.update( + variables=request.json.get("variables", None) + ) + response.set_status(200) + response.json(project) + @Route.get( r"/projects/{project_id}", description="Get project information", diff --git a/tests/compute/test_project.py b/tests/compute/test_project.py index 5713ebf4..82493f8e 100644 --- a/tests/compute/test_project.py +++ b/tests/compute/test_project.py @@ -205,3 +205,10 @@ def test_emit(async_run): (action, event, context) = async_run(queue.get(0.5)) assert action == "test" assert context["project_id"] == project.id + + +def test_update_project(loop): + variables = [{"name": "TEST", "value": "VAL"}] + project = Project(project_id=str(uuid.uuid4())) + loop.run_until_complete(asyncio.async(project.update(variables=variables))) + assert project.variables == variables diff --git a/tests/controller/test_project.py b/tests/controller/test_project.py index 1e76d39e..6a0177e2 100644 --- a/tests/controller/test_project.py +++ b/tests/controller/test_project.py @@ -85,13 +85,27 @@ def test_json(tmpdir): def test_update(controller, async_run): project = Project(controller=controller, name="Hello") controller._notification = MagicMock() - assert project.name == "Hello" async_run(project.update(name="World")) assert project.name == "World" controller.notification.emit.assert_any_call("project.updated", project.__json__()) +def test_update_on_compute(controller, async_run): + variables = [{"name": "TEST", "value": "VAL1"}] + compute = MagicMock() + compute.id = "local" + project = Project(controller=controller, name="Test") + project._project_created_on_compute = [compute] + controller._notification = MagicMock() + + async_run(project.update(variables=variables)) + + compute.put.assert_any_call('/projects/{}'.format(project.id), { + "variables": variables + }) + + def test_path(tmpdir): directory = Config.instance().get_section_config("Server").get("projects_path") @@ -150,7 +164,8 @@ def test_add_node_local(async_run, controller): compute.post.assert_any_call('/projects', data={ "name": project._name, "project_id": project._id, - "path": project._path + "path": project._path, + "variables": None }) compute.post.assert_any_call('/projects/{}/vpcs/nodes'.format(project.id), data={'node_id': node.id, @@ -178,7 +193,8 @@ def test_add_node_non_local(async_run, controller): compute.post.assert_any_call('/projects', data={ "name": project._name, - "project_id": project._id + "project_id": project._id, + "variables": None }) compute.post.assert_any_call('/projects/{}/vpcs/nodes'.format(project.id), data={'node_id': node.id, @@ -218,7 +234,8 @@ def test_add_node_from_appliance(async_run, controller): compute.post.assert_any_call('/projects', data={ "name": project._name, "project_id": project._id, - "path": project._path + "path": project._path, + "variables": None }) compute.post.assert_any_call('/projects/{}/vpcs/nodes'.format(project.id), data={'node_id': node.id, diff --git a/tests/controller/test_topology.py b/tests/controller/test_topology.py index 0bf8a0cb..81eb0a0f 100644 --- a/tests/controller/test_topology.py +++ b/tests/controller/test_topology.py @@ -53,6 +53,8 @@ def test_project_to_topology_empty(tmpdir): "drawings": [] }, "type": "topology", + "supplier": None, + "variables": None, "version": __version__ } diff --git a/tests/handlers/api/compute/test_project.py b/tests/handlers/api/compute/test_project.py index 9db5350a..1fc92e85 100644 --- a/tests/handlers/api/compute/test_project.py +++ b/tests/handlers/api/compute/test_project.py @@ -91,6 +91,23 @@ def test_delete_project(http_compute, project): assert mock.called +def test_update_project(http_compute): + query = {"name": "test", "project_id": "51010203-0405-0607-0809-0a0b0c0d0e0f"} + response = http_compute.post("/projects", query) + assert response.status == 201 + + query = { + "variables": [{"name": "TEST1", "value": "VAL1"}] + } + response = http_compute.put( + "/projects/{project_id}".format(project_id="51010203-0405-0607-0809-0a0b0c0d0e0f"), + query, + example=True + ) + assert response.status == 200 + assert response.json["variables"] == [{"name": "TEST1", "value": "VAL1"}] + + def test_delete_project_invalid_uuid(http_compute): response = http_compute.delete("/projects/{project_id}".format(project_id=uuid.uuid4())) assert response.status == 404 From f3c6faed75f3931890d3bcec7b88166f467cc920 Mon Sep 17 00:00:00 2001 From: ziajka Date: Wed, 9 May 2018 15:53:50 +0200 Subject: [PATCH 10/10] Removed unnecessary line --- gns3server/controller/compute.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gns3server/controller/compute.py b/gns3server/controller/compute.py index 3bdee2f9..e5b8ef93 100644 --- a/gns3server/controller/compute.py +++ b/gns3server/controller/compute.py @@ -464,8 +464,6 @@ class Compute: self._cpu_usage_percent = event["cpu_usage_percent"] self._memory_usage_percent = event["memory_usage_percent"] self._controller.notification.emit("compute.updated", self.__json__()) - elif action == 'project.updated': - print(event) else: yield from self._controller.notification.dispatch(action, event, compute_id=self.id) if self._ws: